1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (C) 2013 Emulex 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Emulex Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 * 33 * Contact Information: 34 * freebsd-drivers@emulex.com 35 * 36 * Emulex 37 * 3333 Susan Street 38 * Costa Mesa, CA 92626 39 */ 40 41 /* $FreeBSD$ */ 42 43 #include "opt_inet6.h" 44 #include "opt_inet.h" 45 46 #include "oce_if.h" 47 #include "oce_user.h" 48 49 #define is_tso_pkt(m) (m->m_pkthdr.csum_flags & CSUM_TSO) 50 51 /* UE Status Low CSR */ 52 static char *ue_status_low_desc[] = { 53 "CEV", 54 "CTX", 55 "DBUF", 56 "ERX", 57 "Host", 58 "MPU", 59 "NDMA", 60 "PTC ", 61 "RDMA ", 62 "RXF ", 63 "RXIPS ", 64 "RXULP0 ", 65 "RXULP1 ", 66 "RXULP2 ", 67 "TIM ", 68 "TPOST ", 69 "TPRE ", 70 "TXIPS ", 71 "TXULP0 ", 72 "TXULP1 ", 73 "UC ", 74 "WDMA ", 75 "TXULP2 ", 76 "HOST1 ", 77 "P0_OB_LINK ", 78 "P1_OB_LINK ", 79 "HOST_GPIO ", 80 "MBOX ", 81 "AXGMAC0", 82 "AXGMAC1", 83 "JTAG", 84 "MPU_INTPEND" 85 }; 86 87 /* UE Status High CSR */ 88 static char *ue_status_hi_desc[] = { 89 "LPCMEMHOST", 90 "MGMT_MAC", 91 "PCS0ONLINE", 92 "MPU_IRAM", 93 "PCS1ONLINE", 94 "PCTL0", 95 "PCTL1", 96 "PMEM", 97 "RR", 98 "TXPB", 99 "RXPP", 100 "XAUI", 101 "TXP", 102 "ARM", 103 "IPC", 104 "HOST2", 105 "HOST3", 106 "HOST4", 107 "HOST5", 108 "HOST6", 109 "HOST7", 110 "HOST8", 111 "HOST9", 112 "NETC", 113 "Unknown", 114 "Unknown", 115 "Unknown", 116 "Unknown", 117 "Unknown", 118 "Unknown", 119 "Unknown", 120 "Unknown" 121 }; 122 123 struct oce_common_cqe_info{ 124 uint8_t vtp:1; 125 uint8_t l4_cksum_pass:1; 126 uint8_t ip_cksum_pass:1; 127 uint8_t ipv6_frame:1; 128 uint8_t qnq:1; 129 uint8_t rsvd:3; 130 uint8_t num_frags; 131 uint16_t pkt_size; 132 uint16_t vtag; 133 }; 134 135 136 /* Driver entry points prototypes */ 137 static int oce_probe(device_t dev); 138 static int oce_attach(device_t dev); 139 static int oce_detach(device_t dev); 140 static int oce_shutdown(device_t dev); 141 static int oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data); 142 static void oce_init(void *xsc); 143 static int oce_multiq_start(struct ifnet *ifp, struct mbuf *m); 144 static void oce_multiq_flush(struct ifnet *ifp); 145 146 /* Driver interrupt routines protypes */ 147 static void oce_intr(void *arg, int pending); 148 static int oce_setup_intr(POCE_SOFTC sc); 149 static int oce_fast_isr(void *arg); 150 static int oce_alloc_intr(POCE_SOFTC sc, int vector, 151 void (*isr) (void *arg, int pending)); 152 153 /* Media callbacks prototypes */ 154 static void oce_media_status(struct ifnet *ifp, struct ifmediareq *req); 155 static int oce_media_change(struct ifnet *ifp); 156 157 /* Transmit routines prototypes */ 158 static int oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index); 159 static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq); 160 static void oce_process_tx_completion(struct oce_wq *wq); 161 static int oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, 162 struct oce_wq *wq); 163 164 /* Receive routines prototypes */ 165 static int oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe); 166 static int oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe); 167 static void oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe); 168 static void oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq); 169 static uint16_t oce_rq_handler_lro(void *arg); 170 static void oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2); 171 static void oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2); 172 static void oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m); 173 174 /* Helper function prototypes in this file */ 175 static int oce_attach_ifp(POCE_SOFTC sc); 176 static void oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag); 177 static void oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag); 178 static int oce_vid_config(POCE_SOFTC sc); 179 static void oce_mac_addr_set(POCE_SOFTC sc); 180 static int oce_handle_passthrough(struct ifnet *ifp, caddr_t data); 181 static void oce_local_timer(void *arg); 182 static void oce_if_deactivate(POCE_SOFTC sc); 183 static void oce_if_activate(POCE_SOFTC sc); 184 static void setup_max_queues_want(POCE_SOFTC sc); 185 static void update_queues_got(POCE_SOFTC sc); 186 static void process_link_state(POCE_SOFTC sc, 187 struct oce_async_cqe_link_state *acqe); 188 static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m); 189 static void oce_get_config(POCE_SOFTC sc); 190 static struct mbuf *oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete); 191 static void oce_read_env_variables(POCE_SOFTC sc); 192 193 194 /* IP specific */ 195 #if defined(INET6) || defined(INET) 196 static int oce_init_lro(POCE_SOFTC sc); 197 static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp); 198 #endif 199 200 static device_method_t oce_dispatch[] = { 201 DEVMETHOD(device_probe, oce_probe), 202 DEVMETHOD(device_attach, oce_attach), 203 DEVMETHOD(device_detach, oce_detach), 204 DEVMETHOD(device_shutdown, oce_shutdown), 205 206 DEVMETHOD_END 207 }; 208 209 static driver_t oce_driver = { 210 "oce", 211 oce_dispatch, 212 sizeof(OCE_SOFTC) 213 }; 214 static devclass_t oce_devclass; 215 216 217 DRIVER_MODULE(oce, pci, oce_driver, oce_devclass, 0, 0); 218 MODULE_DEPEND(oce, pci, 1, 1, 1); 219 MODULE_DEPEND(oce, ether, 1, 1, 1); 220 MODULE_VERSION(oce, 1); 221 222 223 /* global vars */ 224 const char component_revision[32] = {"///" COMPONENT_REVISION "///"}; 225 226 /* Module capabilites and parameters */ 227 uint32_t oce_max_rsp_handled = OCE_MAX_RSP_HANDLED; 228 uint32_t oce_enable_rss = OCE_MODCAP_RSS; 229 uint32_t oce_rq_buf_size = 2048; 230 231 TUNABLE_INT("hw.oce.max_rsp_handled", &oce_max_rsp_handled); 232 TUNABLE_INT("hw.oce.enable_rss", &oce_enable_rss); 233 234 235 /* Supported devices table */ 236 static uint32_t supportedDevices[] = { 237 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE2, 238 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE3, 239 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_BE3, 240 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201, 241 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201_VF, 242 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_SH 243 }; 244 245 POCE_SOFTC softc_head = NULL; 246 POCE_SOFTC softc_tail = NULL; 247 248 struct oce_rdma_if *oce_rdma_if = NULL; 249 250 /***************************************************************************** 251 * Driver entry points functions * 252 *****************************************************************************/ 253 254 static int 255 oce_probe(device_t dev) 256 { 257 uint16_t vendor = 0; 258 uint16_t device = 0; 259 int i = 0; 260 char str[256] = {0}; 261 POCE_SOFTC sc; 262 263 sc = device_get_softc(dev); 264 bzero(sc, sizeof(OCE_SOFTC)); 265 sc->dev = dev; 266 267 vendor = pci_get_vendor(dev); 268 device = pci_get_device(dev); 269 270 for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint32_t)); i++) { 271 if (vendor == ((supportedDevices[i] >> 16) & 0xffff)) { 272 if (device == (supportedDevices[i] & 0xffff)) { 273 sprintf(str, "%s:%s", "Emulex CNA NIC function", 274 component_revision); 275 device_set_desc_copy(dev, str); 276 277 switch (device) { 278 case PCI_PRODUCT_BE2: 279 sc->flags |= OCE_FLAGS_BE2; 280 break; 281 case PCI_PRODUCT_BE3: 282 sc->flags |= OCE_FLAGS_BE3; 283 break; 284 case PCI_PRODUCT_XE201: 285 case PCI_PRODUCT_XE201_VF: 286 sc->flags |= OCE_FLAGS_XE201; 287 break; 288 case PCI_PRODUCT_SH: 289 sc->flags |= OCE_FLAGS_SH; 290 break; 291 default: 292 return ENXIO; 293 } 294 return BUS_PROBE_DEFAULT; 295 } 296 } 297 } 298 299 return ENXIO; 300 } 301 302 303 static int 304 oce_attach(device_t dev) 305 { 306 POCE_SOFTC sc; 307 int rc = 0; 308 309 sc = device_get_softc(dev); 310 311 rc = oce_hw_pci_alloc(sc); 312 if (rc) 313 return rc; 314 315 sc->tx_ring_size = OCE_TX_RING_SIZE; 316 sc->rx_ring_size = OCE_RX_RING_SIZE; 317 /* receive fragment size should be multiple of 2K */ 318 sc->rq_frag_size = ((oce_rq_buf_size / 2048) * 2048); 319 sc->flow_control = OCE_DEFAULT_FLOW_CONTROL; 320 sc->promisc = OCE_DEFAULT_PROMISCUOUS; 321 322 LOCK_CREATE(&sc->bmbx_lock, "Mailbox_lock"); 323 LOCK_CREATE(&sc->dev_lock, "Device_lock"); 324 325 /* initialise the hardware */ 326 rc = oce_hw_init(sc); 327 if (rc) 328 goto pci_res_free; 329 330 oce_read_env_variables(sc); 331 332 oce_get_config(sc); 333 334 setup_max_queues_want(sc); 335 336 rc = oce_setup_intr(sc); 337 if (rc) 338 goto mbox_free; 339 340 rc = oce_queue_init_all(sc); 341 if (rc) 342 goto intr_free; 343 344 rc = oce_attach_ifp(sc); 345 if (rc) 346 goto queues_free; 347 348 #if defined(INET6) || defined(INET) 349 rc = oce_init_lro(sc); 350 if (rc) 351 goto ifp_free; 352 #endif 353 354 rc = oce_hw_start(sc); 355 if (rc) 356 goto lro_free; 357 358 sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 359 oce_add_vlan, sc, EVENTHANDLER_PRI_FIRST); 360 sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 361 oce_del_vlan, sc, EVENTHANDLER_PRI_FIRST); 362 363 rc = oce_stats_init(sc); 364 if (rc) 365 goto vlan_free; 366 367 oce_add_sysctls(sc); 368 369 callout_init(&sc->timer, CALLOUT_MPSAFE); 370 rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc); 371 if (rc) 372 goto stats_free; 373 374 sc->next =NULL; 375 if (softc_tail != NULL) { 376 softc_tail->next = sc; 377 } else { 378 softc_head = sc; 379 } 380 softc_tail = sc; 381 382 return 0; 383 384 stats_free: 385 callout_drain(&sc->timer); 386 oce_stats_free(sc); 387 vlan_free: 388 if (sc->vlan_attach) 389 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); 390 if (sc->vlan_detach) 391 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); 392 oce_hw_intr_disable(sc); 393 lro_free: 394 #if defined(INET6) || defined(INET) 395 oce_free_lro(sc); 396 ifp_free: 397 #endif 398 ether_ifdetach(sc->ifp); 399 if_free(sc->ifp); 400 queues_free: 401 oce_queue_release_all(sc); 402 intr_free: 403 oce_intr_free(sc); 404 mbox_free: 405 oce_dma_free(sc, &sc->bsmbx); 406 pci_res_free: 407 oce_hw_pci_free(sc); 408 LOCK_DESTROY(&sc->dev_lock); 409 LOCK_DESTROY(&sc->bmbx_lock); 410 return rc; 411 412 } 413 414 415 static int 416 oce_detach(device_t dev) 417 { 418 POCE_SOFTC sc = device_get_softc(dev); 419 POCE_SOFTC poce_sc_tmp, *ppoce_sc_tmp1, poce_sc_tmp2 = NULL; 420 421 poce_sc_tmp = softc_head; 422 ppoce_sc_tmp1 = &softc_head; 423 while (poce_sc_tmp != NULL) { 424 if (poce_sc_tmp == sc) { 425 *ppoce_sc_tmp1 = sc->next; 426 if (sc->next == NULL) { 427 softc_tail = poce_sc_tmp2; 428 } 429 break; 430 } 431 poce_sc_tmp2 = poce_sc_tmp; 432 ppoce_sc_tmp1 = &poce_sc_tmp->next; 433 poce_sc_tmp = poce_sc_tmp->next; 434 } 435 436 LOCK(&sc->dev_lock); 437 oce_if_deactivate(sc); 438 UNLOCK(&sc->dev_lock); 439 440 callout_drain(&sc->timer); 441 442 if (sc->vlan_attach != NULL) 443 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); 444 if (sc->vlan_detach != NULL) 445 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); 446 447 ether_ifdetach(sc->ifp); 448 449 if_free(sc->ifp); 450 451 oce_hw_shutdown(sc); 452 453 bus_generic_detach(dev); 454 455 return 0; 456 } 457 458 459 static int 460 oce_shutdown(device_t dev) 461 { 462 int rc; 463 464 rc = oce_detach(dev); 465 466 return rc; 467 } 468 469 470 static int 471 oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 472 { 473 struct ifreq *ifr = (struct ifreq *)data; 474 POCE_SOFTC sc = ifp->if_softc; 475 int rc = 0; 476 uint32_t u; 477 478 switch (command) { 479 480 case SIOCGIFMEDIA: 481 rc = ifmedia_ioctl(ifp, ifr, &sc->media, command); 482 break; 483 484 case SIOCSIFMTU: 485 if (ifr->ifr_mtu > OCE_MAX_MTU) 486 rc = EINVAL; 487 else 488 ifp->if_mtu = ifr->ifr_mtu; 489 break; 490 491 case SIOCSIFFLAGS: 492 if (ifp->if_flags & IFF_UP) { 493 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 494 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; 495 oce_init(sc); 496 } 497 device_printf(sc->dev, "Interface Up\n"); 498 } else { 499 LOCK(&sc->dev_lock); 500 501 sc->ifp->if_drv_flags &= 502 ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 503 oce_if_deactivate(sc); 504 505 UNLOCK(&sc->dev_lock); 506 507 device_printf(sc->dev, "Interface Down\n"); 508 } 509 510 if ((ifp->if_flags & IFF_PROMISC) && !sc->promisc) { 511 if (!oce_rxf_set_promiscuous(sc, (1 | (1 << 1)))) 512 sc->promisc = TRUE; 513 } else if (!(ifp->if_flags & IFF_PROMISC) && sc->promisc) { 514 if (!oce_rxf_set_promiscuous(sc, 0)) 515 sc->promisc = FALSE; 516 } 517 518 break; 519 520 case SIOCADDMULTI: 521 case SIOCDELMULTI: 522 rc = oce_hw_update_multicast(sc); 523 if (rc) 524 device_printf(sc->dev, 525 "Update multicast address failed\n"); 526 break; 527 528 case SIOCSIFCAP: 529 u = ifr->ifr_reqcap ^ ifp->if_capenable; 530 531 if (u & IFCAP_TXCSUM) { 532 ifp->if_capenable ^= IFCAP_TXCSUM; 533 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 534 535 if (IFCAP_TSO & ifp->if_capenable && 536 !(IFCAP_TXCSUM & ifp->if_capenable)) { 537 ifp->if_capenable &= ~IFCAP_TSO; 538 ifp->if_hwassist &= ~CSUM_TSO; 539 if_printf(ifp, 540 "TSO disabled due to -txcsum.\n"); 541 } 542 } 543 544 if (u & IFCAP_RXCSUM) 545 ifp->if_capenable ^= IFCAP_RXCSUM; 546 547 if (u & IFCAP_TSO4) { 548 ifp->if_capenable ^= IFCAP_TSO4; 549 550 if (IFCAP_TSO & ifp->if_capenable) { 551 if (IFCAP_TXCSUM & ifp->if_capenable) 552 ifp->if_hwassist |= CSUM_TSO; 553 else { 554 ifp->if_capenable &= ~IFCAP_TSO; 555 ifp->if_hwassist &= ~CSUM_TSO; 556 if_printf(ifp, 557 "Enable txcsum first.\n"); 558 rc = EAGAIN; 559 } 560 } else 561 ifp->if_hwassist &= ~CSUM_TSO; 562 } 563 564 if (u & IFCAP_VLAN_HWTAGGING) 565 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 566 567 if (u & IFCAP_VLAN_HWFILTER) { 568 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 569 oce_vid_config(sc); 570 } 571 #if defined(INET6) || defined(INET) 572 if (u & IFCAP_LRO) { 573 ifp->if_capenable ^= IFCAP_LRO; 574 if(sc->enable_hwlro) { 575 if(ifp->if_capenable & IFCAP_LRO) { 576 rc = oce_mbox_nic_set_iface_lro_config(sc, 1); 577 }else { 578 rc = oce_mbox_nic_set_iface_lro_config(sc, 0); 579 } 580 } 581 } 582 #endif 583 584 break; 585 586 case SIOCGPRIVATE_0: 587 rc = oce_handle_passthrough(ifp, data); 588 break; 589 default: 590 rc = ether_ioctl(ifp, command, data); 591 break; 592 } 593 594 return rc; 595 } 596 597 598 static void 599 oce_init(void *arg) 600 { 601 POCE_SOFTC sc = arg; 602 603 LOCK(&sc->dev_lock); 604 605 if (sc->ifp->if_flags & IFF_UP) { 606 oce_if_deactivate(sc); 607 oce_if_activate(sc); 608 } 609 610 UNLOCK(&sc->dev_lock); 611 612 } 613 614 615 static int 616 oce_multiq_start(struct ifnet *ifp, struct mbuf *m) 617 { 618 POCE_SOFTC sc = ifp->if_softc; 619 struct oce_wq *wq = NULL; 620 int queue_index = 0; 621 int status = 0; 622 623 if (!sc->link_status) 624 return ENXIO; 625 626 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 627 queue_index = m->m_pkthdr.flowid % sc->nwqs; 628 629 wq = sc->wq[queue_index]; 630 631 LOCK(&wq->tx_lock); 632 status = oce_multiq_transmit(ifp, m, wq); 633 UNLOCK(&wq->tx_lock); 634 635 return status; 636 637 } 638 639 640 static void 641 oce_multiq_flush(struct ifnet *ifp) 642 { 643 POCE_SOFTC sc = ifp->if_softc; 644 struct mbuf *m; 645 int i = 0; 646 647 for (i = 0; i < sc->nwqs; i++) { 648 while ((m = buf_ring_dequeue_sc(sc->wq[i]->br)) != NULL) 649 m_freem(m); 650 } 651 if_qflush(ifp); 652 } 653 654 655 656 /***************************************************************************** 657 * Driver interrupt routines functions * 658 *****************************************************************************/ 659 660 static void 661 oce_intr(void *arg, int pending) 662 { 663 664 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg; 665 POCE_SOFTC sc = ii->sc; 666 struct oce_eq *eq = ii->eq; 667 struct oce_eqe *eqe; 668 struct oce_cq *cq = NULL; 669 int i, num_eqes = 0; 670 671 672 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, 673 BUS_DMASYNC_POSTWRITE); 674 do { 675 eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe); 676 if (eqe->evnt == 0) 677 break; 678 eqe->evnt = 0; 679 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, 680 BUS_DMASYNC_POSTWRITE); 681 RING_GET(eq->ring, 1); 682 num_eqes++; 683 684 } while (TRUE); 685 686 if (!num_eqes) 687 goto eq_arm; /* Spurious */ 688 689 /* Clear EQ entries, but dont arm */ 690 oce_arm_eq(sc, eq->eq_id, num_eqes, FALSE, FALSE); 691 692 /* Process TX, RX and MCC. But dont arm CQ*/ 693 for (i = 0; i < eq->cq_valid; i++) { 694 cq = eq->cq[i]; 695 (*cq->cq_handler)(cq->cb_arg); 696 } 697 698 /* Arm all cqs connected to this EQ */ 699 for (i = 0; i < eq->cq_valid; i++) { 700 cq = eq->cq[i]; 701 oce_arm_cq(sc, cq->cq_id, 0, TRUE); 702 } 703 704 eq_arm: 705 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); 706 707 return; 708 } 709 710 711 static int 712 oce_setup_intr(POCE_SOFTC sc) 713 { 714 int rc = 0, use_intx = 0; 715 int vector = 0, req_vectors = 0; 716 int tot_req_vectors, tot_vectors; 717 718 if (is_rss_enabled(sc)) 719 req_vectors = MAX((sc->nrqs - 1), sc->nwqs); 720 else 721 req_vectors = 1; 722 723 tot_req_vectors = req_vectors; 724 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 725 if (req_vectors > 1) { 726 tot_req_vectors += OCE_RDMA_VECTORS; 727 sc->roce_intr_count = OCE_RDMA_VECTORS; 728 } 729 } 730 731 if (sc->flags & OCE_FLAGS_MSIX_CAPABLE) { 732 sc->intr_count = req_vectors; 733 tot_vectors = tot_req_vectors; 734 rc = pci_alloc_msix(sc->dev, &tot_vectors); 735 if (rc != 0) { 736 use_intx = 1; 737 pci_release_msi(sc->dev); 738 } else { 739 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 740 if (tot_vectors < tot_req_vectors) { 741 if (sc->intr_count < (2 * OCE_RDMA_VECTORS)) { 742 sc->roce_intr_count = (tot_vectors / 2); 743 } 744 sc->intr_count = tot_vectors - sc->roce_intr_count; 745 } 746 } else { 747 sc->intr_count = tot_vectors; 748 } 749 sc->flags |= OCE_FLAGS_USING_MSIX; 750 } 751 } else 752 use_intx = 1; 753 754 if (use_intx) 755 sc->intr_count = 1; 756 757 /* Scale number of queues based on intr we got */ 758 update_queues_got(sc); 759 760 if (use_intx) { 761 device_printf(sc->dev, "Using legacy interrupt\n"); 762 rc = oce_alloc_intr(sc, vector, oce_intr); 763 if (rc) 764 goto error; 765 } else { 766 for (; vector < sc->intr_count; vector++) { 767 rc = oce_alloc_intr(sc, vector, oce_intr); 768 if (rc) 769 goto error; 770 } 771 } 772 773 return 0; 774 error: 775 oce_intr_free(sc); 776 return rc; 777 } 778 779 780 static int 781 oce_fast_isr(void *arg) 782 { 783 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg; 784 POCE_SOFTC sc = ii->sc; 785 786 if (ii->eq == NULL) 787 return FILTER_STRAY; 788 789 oce_arm_eq(sc, ii->eq->eq_id, 0, FALSE, TRUE); 790 791 taskqueue_enqueue(ii->tq, &ii->task); 792 793 ii->eq->intr++; 794 795 return FILTER_HANDLED; 796 } 797 798 799 static int 800 oce_alloc_intr(POCE_SOFTC sc, int vector, void (*isr) (void *arg, int pending)) 801 { 802 POCE_INTR_INFO ii = &sc->intrs[vector]; 803 int rc = 0, rr; 804 805 if (vector >= OCE_MAX_EQ) 806 return (EINVAL); 807 808 /* Set the resource id for the interrupt. 809 * MSIx is vector + 1 for the resource id, 810 * INTx is 0 for the resource id. 811 */ 812 if (sc->flags & OCE_FLAGS_USING_MSIX) 813 rr = vector + 1; 814 else 815 rr = 0; 816 ii->intr_res = bus_alloc_resource_any(sc->dev, 817 SYS_RES_IRQ, 818 &rr, RF_ACTIVE|RF_SHAREABLE); 819 ii->irq_rr = rr; 820 if (ii->intr_res == NULL) { 821 device_printf(sc->dev, 822 "Could not allocate interrupt\n"); 823 rc = ENXIO; 824 return rc; 825 } 826 827 TASK_INIT(&ii->task, 0, isr, ii); 828 ii->vector = vector; 829 sprintf(ii->task_name, "oce_task[%d]", ii->vector); 830 ii->tq = taskqueue_create_fast(ii->task_name, 831 M_NOWAIT, 832 taskqueue_thread_enqueue, 833 &ii->tq); 834 taskqueue_start_threads(&ii->tq, 1, PI_NET, "%s taskq", 835 device_get_nameunit(sc->dev)); 836 837 ii->sc = sc; 838 rc = bus_setup_intr(sc->dev, 839 ii->intr_res, 840 INTR_TYPE_NET, 841 oce_fast_isr, NULL, ii, &ii->tag); 842 return rc; 843 844 } 845 846 847 void 848 oce_intr_free(POCE_SOFTC sc) 849 { 850 int i = 0; 851 852 for (i = 0; i < sc->intr_count; i++) { 853 854 if (sc->intrs[i].tag != NULL) 855 bus_teardown_intr(sc->dev, sc->intrs[i].intr_res, 856 sc->intrs[i].tag); 857 if (sc->intrs[i].tq != NULL) 858 taskqueue_free(sc->intrs[i].tq); 859 860 if (sc->intrs[i].intr_res != NULL) 861 bus_release_resource(sc->dev, SYS_RES_IRQ, 862 sc->intrs[i].irq_rr, 863 sc->intrs[i].intr_res); 864 sc->intrs[i].tag = NULL; 865 sc->intrs[i].intr_res = NULL; 866 } 867 868 if (sc->flags & OCE_FLAGS_USING_MSIX) 869 pci_release_msi(sc->dev); 870 871 } 872 873 874 875 /****************************************************************************** 876 * Media callbacks functions * 877 ******************************************************************************/ 878 879 static void 880 oce_media_status(struct ifnet *ifp, struct ifmediareq *req) 881 { 882 POCE_SOFTC sc = (POCE_SOFTC) ifp->if_softc; 883 884 885 req->ifm_status = IFM_AVALID; 886 req->ifm_active = IFM_ETHER; 887 888 if (sc->link_status == 1) 889 req->ifm_status |= IFM_ACTIVE; 890 else 891 return; 892 893 switch (sc->link_speed) { 894 case 1: /* 10 Mbps */ 895 req->ifm_active |= IFM_10_T | IFM_FDX; 896 sc->speed = 10; 897 break; 898 case 2: /* 100 Mbps */ 899 req->ifm_active |= IFM_100_TX | IFM_FDX; 900 sc->speed = 100; 901 break; 902 case 3: /* 1 Gbps */ 903 req->ifm_active |= IFM_1000_T | IFM_FDX; 904 sc->speed = 1000; 905 break; 906 case 4: /* 10 Gbps */ 907 req->ifm_active |= IFM_10G_SR | IFM_FDX; 908 sc->speed = 10000; 909 break; 910 case 5: /* 20 Gbps */ 911 req->ifm_active |= IFM_10G_SR | IFM_FDX; 912 sc->speed = 20000; 913 break; 914 case 6: /* 25 Gbps */ 915 req->ifm_active |= IFM_10G_SR | IFM_FDX; 916 sc->speed = 25000; 917 break; 918 case 7: /* 40 Gbps */ 919 req->ifm_active |= IFM_40G_SR4 | IFM_FDX; 920 sc->speed = 40000; 921 break; 922 default: 923 sc->speed = 0; 924 break; 925 } 926 927 return; 928 } 929 930 931 int 932 oce_media_change(struct ifnet *ifp) 933 { 934 return 0; 935 } 936 937 938 static void oce_is_pkt_dest_bmc(POCE_SOFTC sc, 939 struct mbuf *m, boolean_t *os2bmc, 940 struct mbuf **m_new) 941 { 942 struct ether_header *eh = NULL; 943 944 eh = mtod(m, struct ether_header *); 945 946 if (!is_os2bmc_enabled(sc) || *os2bmc) { 947 *os2bmc = FALSE; 948 goto done; 949 } 950 if (!ETHER_IS_MULTICAST(eh->ether_dhost)) 951 goto done; 952 953 if (is_mc_allowed_on_bmc(sc, eh) || 954 is_bc_allowed_on_bmc(sc, eh) || 955 is_arp_allowed_on_bmc(sc, ntohs(eh->ether_type))) { 956 *os2bmc = TRUE; 957 goto done; 958 } 959 960 if (mtod(m, struct ip *)->ip_p == IPPROTO_IPV6) { 961 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); 962 uint8_t nexthdr = ip6->ip6_nxt; 963 if (nexthdr == IPPROTO_ICMPV6) { 964 struct icmp6_hdr *icmp6 = (struct icmp6_hdr *)(ip6 + 1); 965 switch (icmp6->icmp6_type) { 966 case ND_ROUTER_ADVERT: 967 *os2bmc = is_ipv6_ra_filt_enabled(sc); 968 goto done; 969 case ND_NEIGHBOR_ADVERT: 970 *os2bmc = is_ipv6_na_filt_enabled(sc); 971 goto done; 972 default: 973 break; 974 } 975 } 976 } 977 978 if (mtod(m, struct ip *)->ip_p == IPPROTO_UDP) { 979 struct ip *ip = mtod(m, struct ip *); 980 int iphlen = ip->ip_hl << 2; 981 struct udphdr *uh = (struct udphdr *)((caddr_t)ip + iphlen); 982 switch (uh->uh_dport) { 983 case DHCP_CLIENT_PORT: 984 *os2bmc = is_dhcp_client_filt_enabled(sc); 985 goto done; 986 case DHCP_SERVER_PORT: 987 *os2bmc = is_dhcp_srvr_filt_enabled(sc); 988 goto done; 989 case NET_BIOS_PORT1: 990 case NET_BIOS_PORT2: 991 *os2bmc = is_nbios_filt_enabled(sc); 992 goto done; 993 case DHCPV6_RAS_PORT: 994 *os2bmc = is_ipv6_ras_filt_enabled(sc); 995 goto done; 996 default: 997 break; 998 } 999 } 1000 done: 1001 if (*os2bmc) { 1002 *m_new = m_dup(m, M_NOWAIT); 1003 if (!*m_new) { 1004 *os2bmc = FALSE; 1005 return; 1006 } 1007 *m_new = oce_insert_vlan_tag(sc, *m_new, NULL); 1008 } 1009 } 1010 1011 1012 1013 /***************************************************************************** 1014 * Transmit routines functions * 1015 *****************************************************************************/ 1016 1017 static int 1018 oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index) 1019 { 1020 int rc = 0, i, retry_cnt = 0; 1021 bus_dma_segment_t segs[OCE_MAX_TX_ELEMENTS]; 1022 struct mbuf *m, *m_temp, *m_new = NULL; 1023 struct oce_wq *wq = sc->wq[wq_index]; 1024 struct oce_packet_desc *pd; 1025 struct oce_nic_hdr_wqe *nichdr; 1026 struct oce_nic_frag_wqe *nicfrag; 1027 struct ether_header *eh = NULL; 1028 int num_wqes; 1029 uint32_t reg_value; 1030 boolean_t complete = TRUE; 1031 boolean_t os2bmc = FALSE; 1032 1033 m = *mpp; 1034 if (!m) 1035 return EINVAL; 1036 1037 if (!(m->m_flags & M_PKTHDR)) { 1038 rc = ENXIO; 1039 goto free_ret; 1040 } 1041 1042 /* Don't allow non-TSO packets longer than MTU */ 1043 if (!is_tso_pkt(m)) { 1044 eh = mtod(m, struct ether_header *); 1045 if(m->m_pkthdr.len > ETHER_MAX_FRAME(sc->ifp, eh->ether_type, FALSE)) 1046 goto free_ret; 1047 } 1048 1049 if(oce_tx_asic_stall_verify(sc, m)) { 1050 m = oce_insert_vlan_tag(sc, m, &complete); 1051 if(!m) { 1052 device_printf(sc->dev, "Insertion unsuccessful\n"); 1053 return 0; 1054 } 1055 1056 } 1057 1058 /* Lancer, SH ASIC has a bug wherein Packets that are 32 bytes or less 1059 * may cause a transmit stall on that port. So the work-around is to 1060 * pad short packets (<= 32 bytes) to a 36-byte length. 1061 */ 1062 if(IS_SH(sc) || IS_XE201(sc) ) { 1063 if(m->m_pkthdr.len <= 32) { 1064 char buf[36]; 1065 bzero((void *)buf, 36); 1066 m_append(m, (36 - m->m_pkthdr.len), buf); 1067 } 1068 } 1069 1070 tx_start: 1071 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1072 /* consolidate packet buffers for TSO/LSO segment offload */ 1073 #if defined(INET6) || defined(INET) 1074 m = oce_tso_setup(sc, mpp); 1075 #else 1076 m = NULL; 1077 #endif 1078 if (m == NULL) { 1079 rc = ENXIO; 1080 goto free_ret; 1081 } 1082 } 1083 1084 1085 pd = &wq->pckts[wq->pkt_desc_head]; 1086 1087 retry: 1088 rc = bus_dmamap_load_mbuf_sg(wq->tag, 1089 pd->map, 1090 m, segs, &pd->nsegs, BUS_DMA_NOWAIT); 1091 if (rc == 0) { 1092 num_wqes = pd->nsegs + 1; 1093 if (IS_BE(sc) || IS_SH(sc)) { 1094 /*Dummy required only for BE3.*/ 1095 if (num_wqes & 1) 1096 num_wqes++; 1097 } 1098 if (num_wqes >= RING_NUM_FREE(wq->ring)) { 1099 bus_dmamap_unload(wq->tag, pd->map); 1100 return EBUSY; 1101 } 1102 atomic_store_rel_int(&wq->pkt_desc_head, 1103 (wq->pkt_desc_head + 1) % \ 1104 OCE_WQ_PACKET_ARRAY_SIZE); 1105 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_PREWRITE); 1106 pd->mbuf = m; 1107 1108 nichdr = 1109 RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_hdr_wqe); 1110 nichdr->u0.dw[0] = 0; 1111 nichdr->u0.dw[1] = 0; 1112 nichdr->u0.dw[2] = 0; 1113 nichdr->u0.dw[3] = 0; 1114 1115 nichdr->u0.s.complete = complete; 1116 nichdr->u0.s.mgmt = os2bmc; 1117 nichdr->u0.s.event = 1; 1118 nichdr->u0.s.crc = 1; 1119 nichdr->u0.s.forward = 0; 1120 nichdr->u0.s.ipcs = (m->m_pkthdr.csum_flags & CSUM_IP) ? 1 : 0; 1121 nichdr->u0.s.udpcs = 1122 (m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0; 1123 nichdr->u0.s.tcpcs = 1124 (m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0; 1125 nichdr->u0.s.num_wqe = num_wqes; 1126 nichdr->u0.s.total_length = m->m_pkthdr.len; 1127 1128 if (m->m_flags & M_VLANTAG) { 1129 nichdr->u0.s.vlan = 1; /*Vlan present*/ 1130 nichdr->u0.s.vlan_tag = m->m_pkthdr.ether_vtag; 1131 } 1132 1133 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1134 if (m->m_pkthdr.tso_segsz) { 1135 nichdr->u0.s.lso = 1; 1136 nichdr->u0.s.lso_mss = m->m_pkthdr.tso_segsz; 1137 } 1138 if (!IS_BE(sc) || !IS_SH(sc)) 1139 nichdr->u0.s.ipcs = 1; 1140 } 1141 1142 RING_PUT(wq->ring, 1); 1143 atomic_add_int(&wq->ring->num_used, 1); 1144 1145 for (i = 0; i < pd->nsegs; i++) { 1146 nicfrag = 1147 RING_GET_PRODUCER_ITEM_VA(wq->ring, 1148 struct oce_nic_frag_wqe); 1149 nicfrag->u0.s.rsvd0 = 0; 1150 nicfrag->u0.s.frag_pa_hi = ADDR_HI(segs[i].ds_addr); 1151 nicfrag->u0.s.frag_pa_lo = ADDR_LO(segs[i].ds_addr); 1152 nicfrag->u0.s.frag_len = segs[i].ds_len; 1153 pd->wqe_idx = wq->ring->pidx; 1154 RING_PUT(wq->ring, 1); 1155 atomic_add_int(&wq->ring->num_used, 1); 1156 } 1157 if (num_wqes > (pd->nsegs + 1)) { 1158 nicfrag = 1159 RING_GET_PRODUCER_ITEM_VA(wq->ring, 1160 struct oce_nic_frag_wqe); 1161 nicfrag->u0.dw[0] = 0; 1162 nicfrag->u0.dw[1] = 0; 1163 nicfrag->u0.dw[2] = 0; 1164 nicfrag->u0.dw[3] = 0; 1165 pd->wqe_idx = wq->ring->pidx; 1166 RING_PUT(wq->ring, 1); 1167 atomic_add_int(&wq->ring->num_used, 1); 1168 pd->nsegs++; 1169 } 1170 1171 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1); 1172 wq->tx_stats.tx_reqs++; 1173 wq->tx_stats.tx_wrbs += num_wqes; 1174 wq->tx_stats.tx_bytes += m->m_pkthdr.len; 1175 wq->tx_stats.tx_pkts++; 1176 1177 bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map, 1178 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1179 reg_value = (num_wqes << 16) | wq->wq_id; 1180 1181 /* if os2bmc is not enabled or if the pkt is already tagged as 1182 bmc, do nothing 1183 */ 1184 oce_is_pkt_dest_bmc(sc, m, &os2bmc, &m_new); 1185 1186 OCE_WRITE_REG32(sc, db, wq->db_offset, reg_value); 1187 1188 } else if (rc == EFBIG) { 1189 if (retry_cnt == 0) { 1190 m_temp = m_defrag(m, M_NOWAIT); 1191 if (m_temp == NULL) 1192 goto free_ret; 1193 m = m_temp; 1194 *mpp = m_temp; 1195 retry_cnt = retry_cnt + 1; 1196 goto retry; 1197 } else 1198 goto free_ret; 1199 } else if (rc == ENOMEM) 1200 return rc; 1201 else 1202 goto free_ret; 1203 1204 if (os2bmc) { 1205 m = m_new; 1206 goto tx_start; 1207 } 1208 1209 return 0; 1210 1211 free_ret: 1212 m_freem(*mpp); 1213 *mpp = NULL; 1214 return rc; 1215 } 1216 1217 1218 static void 1219 oce_process_tx_completion(struct oce_wq *wq) 1220 { 1221 struct oce_packet_desc *pd; 1222 POCE_SOFTC sc = (POCE_SOFTC) wq->parent; 1223 struct mbuf *m; 1224 1225 pd = &wq->pckts[wq->pkt_desc_tail]; 1226 atomic_store_rel_int(&wq->pkt_desc_tail, 1227 (wq->pkt_desc_tail + 1) % OCE_WQ_PACKET_ARRAY_SIZE); 1228 atomic_subtract_int(&wq->ring->num_used, pd->nsegs + 1); 1229 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1230 bus_dmamap_unload(wq->tag, pd->map); 1231 1232 m = pd->mbuf; 1233 m_freem(m); 1234 pd->mbuf = NULL; 1235 1236 1237 if (sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) { 1238 if (wq->ring->num_used < (wq->ring->num_items / 2)) { 1239 sc->ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE); 1240 oce_tx_restart(sc, wq); 1241 } 1242 } 1243 } 1244 1245 1246 static void 1247 oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq) 1248 { 1249 1250 if ((sc->ifp->if_drv_flags & IFF_DRV_RUNNING) != IFF_DRV_RUNNING) 1251 return; 1252 1253 #if __FreeBSD_version >= 800000 1254 if (!drbr_empty(sc->ifp, wq->br)) 1255 #else 1256 if (!IFQ_DRV_IS_EMPTY(&sc->ifp->if_snd)) 1257 #endif 1258 taskqueue_enqueue(taskqueue_swi, &wq->txtask); 1259 1260 } 1261 1262 1263 #if defined(INET6) || defined(INET) 1264 static struct mbuf * 1265 oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp) 1266 { 1267 struct mbuf *m; 1268 #ifdef INET 1269 struct ip *ip; 1270 #endif 1271 #ifdef INET6 1272 struct ip6_hdr *ip6; 1273 #endif 1274 struct ether_vlan_header *eh; 1275 struct tcphdr *th; 1276 uint16_t etype; 1277 int total_len = 0, ehdrlen = 0; 1278 1279 m = *mpp; 1280 1281 if (M_WRITABLE(m) == 0) { 1282 m = m_dup(*mpp, M_NOWAIT); 1283 if (!m) 1284 return NULL; 1285 m_freem(*mpp); 1286 *mpp = m; 1287 } 1288 1289 eh = mtod(m, struct ether_vlan_header *); 1290 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 1291 etype = ntohs(eh->evl_proto); 1292 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 1293 } else { 1294 etype = ntohs(eh->evl_encap_proto); 1295 ehdrlen = ETHER_HDR_LEN; 1296 } 1297 1298 switch (etype) { 1299 #ifdef INET 1300 case ETHERTYPE_IP: 1301 ip = (struct ip *)(m->m_data + ehdrlen); 1302 if (ip->ip_p != IPPROTO_TCP) 1303 return NULL; 1304 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2)); 1305 1306 total_len = ehdrlen + (ip->ip_hl << 2) + (th->th_off << 2); 1307 break; 1308 #endif 1309 #ifdef INET6 1310 case ETHERTYPE_IPV6: 1311 ip6 = (struct ip6_hdr *)(m->m_data + ehdrlen); 1312 if (ip6->ip6_nxt != IPPROTO_TCP) 1313 return NULL; 1314 th = (struct tcphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr)); 1315 1316 total_len = ehdrlen + sizeof(struct ip6_hdr) + (th->th_off << 2); 1317 break; 1318 #endif 1319 default: 1320 return NULL; 1321 } 1322 1323 m = m_pullup(m, total_len); 1324 if (!m) 1325 return NULL; 1326 *mpp = m; 1327 return m; 1328 1329 } 1330 #endif /* INET6 || INET */ 1331 1332 void 1333 oce_tx_task(void *arg, int npending) 1334 { 1335 struct oce_wq *wq = arg; 1336 POCE_SOFTC sc = wq->parent; 1337 struct ifnet *ifp = sc->ifp; 1338 int rc = 0; 1339 1340 #if __FreeBSD_version >= 800000 1341 LOCK(&wq->tx_lock); 1342 rc = oce_multiq_transmit(ifp, NULL, wq); 1343 if (rc) { 1344 device_printf(sc->dev, 1345 "TX[%d] restart failed\n", wq->queue_index); 1346 } 1347 UNLOCK(&wq->tx_lock); 1348 #else 1349 oce_start(ifp); 1350 #endif 1351 1352 } 1353 1354 1355 void 1356 oce_start(struct ifnet *ifp) 1357 { 1358 POCE_SOFTC sc = ifp->if_softc; 1359 struct mbuf *m; 1360 int rc = 0; 1361 int def_q = 0; /* Defualt tx queue is 0*/ 1362 1363 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 1364 IFF_DRV_RUNNING) 1365 return; 1366 1367 if (!sc->link_status) 1368 return; 1369 1370 do { 1371 IF_DEQUEUE(&sc->ifp->if_snd, m); 1372 if (m == NULL) 1373 break; 1374 1375 LOCK(&sc->wq[def_q]->tx_lock); 1376 rc = oce_tx(sc, &m, def_q); 1377 UNLOCK(&sc->wq[def_q]->tx_lock); 1378 if (rc) { 1379 if (m != NULL) { 1380 sc->wq[def_q]->tx_stats.tx_stops ++; 1381 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1382 IFQ_DRV_PREPEND(&ifp->if_snd, m); 1383 m = NULL; 1384 } 1385 break; 1386 } 1387 if (m != NULL) 1388 ETHER_BPF_MTAP(ifp, m); 1389 1390 } while (TRUE); 1391 1392 return; 1393 } 1394 1395 1396 /* Handle the Completion Queue for transmit */ 1397 uint16_t 1398 oce_wq_handler(void *arg) 1399 { 1400 struct oce_wq *wq = (struct oce_wq *)arg; 1401 POCE_SOFTC sc = wq->parent; 1402 struct oce_cq *cq = wq->cq; 1403 struct oce_nic_tx_cqe *cqe; 1404 int num_cqes = 0; 1405 1406 LOCK(&wq->tx_compl_lock); 1407 bus_dmamap_sync(cq->ring->dma.tag, 1408 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1409 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 1410 while (cqe->u0.dw[3]) { 1411 DW_SWAP((uint32_t *) cqe, sizeof(oce_wq_cqe)); 1412 1413 wq->ring->cidx = cqe->u0.s.wqe_index + 1; 1414 if (wq->ring->cidx >= wq->ring->num_items) 1415 wq->ring->cidx -= wq->ring->num_items; 1416 1417 oce_process_tx_completion(wq); 1418 wq->tx_stats.tx_compl++; 1419 cqe->u0.dw[3] = 0; 1420 RING_GET(cq->ring, 1); 1421 bus_dmamap_sync(cq->ring->dma.tag, 1422 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1423 cqe = 1424 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 1425 num_cqes++; 1426 } 1427 1428 if (num_cqes) 1429 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); 1430 1431 UNLOCK(&wq->tx_compl_lock); 1432 return num_cqes; 1433 } 1434 1435 1436 static int 1437 oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, struct oce_wq *wq) 1438 { 1439 POCE_SOFTC sc = ifp->if_softc; 1440 int status = 0, queue_index = 0; 1441 struct mbuf *next = NULL; 1442 struct buf_ring *br = NULL; 1443 1444 br = wq->br; 1445 queue_index = wq->queue_index; 1446 1447 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 1448 IFF_DRV_RUNNING) { 1449 if (m != NULL) 1450 status = drbr_enqueue(ifp, br, m); 1451 return status; 1452 } 1453 1454 if (m != NULL) { 1455 if ((status = drbr_enqueue(ifp, br, m)) != 0) 1456 return status; 1457 } 1458 while ((next = drbr_peek(ifp, br)) != NULL) { 1459 if (oce_tx(sc, &next, queue_index)) { 1460 if (next == NULL) { 1461 drbr_advance(ifp, br); 1462 } else { 1463 drbr_putback(ifp, br, next); 1464 wq->tx_stats.tx_stops ++; 1465 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1466 } 1467 break; 1468 } 1469 drbr_advance(ifp, br); 1470 if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len); 1471 if (next->m_flags & M_MCAST) 1472 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); 1473 ETHER_BPF_MTAP(ifp, next); 1474 } 1475 1476 return 0; 1477 } 1478 1479 1480 1481 1482 /***************************************************************************** 1483 * Receive routines functions * 1484 *****************************************************************************/ 1485 1486 static void 1487 oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2) 1488 { 1489 uint32_t *p; 1490 struct ether_header *eh = NULL; 1491 struct tcphdr *tcp_hdr = NULL; 1492 struct ip *ip4_hdr = NULL; 1493 struct ip6_hdr *ip6 = NULL; 1494 uint32_t payload_len = 0; 1495 1496 eh = mtod(m, struct ether_header *); 1497 /* correct IP header */ 1498 if(!cqe2->ipv6_frame) { 1499 ip4_hdr = (struct ip *)((char*)eh + sizeof(struct ether_header)); 1500 ip4_hdr->ip_ttl = cqe2->frame_lifespan; 1501 ip4_hdr->ip_len = htons(cqe2->coalesced_size - sizeof(struct ether_header)); 1502 tcp_hdr = (struct tcphdr *)((char*)ip4_hdr + sizeof(struct ip)); 1503 }else { 1504 ip6 = (struct ip6_hdr *)((char*)eh + sizeof(struct ether_header)); 1505 ip6->ip6_ctlun.ip6_un1.ip6_un1_hlim = cqe2->frame_lifespan; 1506 payload_len = cqe2->coalesced_size - sizeof(struct ether_header) 1507 - sizeof(struct ip6_hdr); 1508 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = htons(payload_len); 1509 tcp_hdr = (struct tcphdr *)((char*)ip6 + sizeof(struct ip6_hdr)); 1510 } 1511 1512 /* correct tcp header */ 1513 tcp_hdr->th_ack = htonl(cqe2->tcp_ack_num); 1514 if(cqe2->push) { 1515 tcp_hdr->th_flags |= TH_PUSH; 1516 } 1517 tcp_hdr->th_win = htons(cqe2->tcp_window); 1518 tcp_hdr->th_sum = 0xffff; 1519 if(cqe2->ts_opt) { 1520 p = (uint32_t *)((char*)tcp_hdr + sizeof(struct tcphdr) + 2); 1521 *p = cqe1->tcp_timestamp_val; 1522 *(p+1) = cqe1->tcp_timestamp_ecr; 1523 } 1524 1525 return; 1526 } 1527 1528 static void 1529 oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m) 1530 { 1531 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1532 uint32_t i = 0, frag_len = 0; 1533 uint32_t len = cqe_info->pkt_size; 1534 struct oce_packet_desc *pd; 1535 struct mbuf *tail = NULL; 1536 1537 for (i = 0; i < cqe_info->num_frags; i++) { 1538 if (rq->ring->cidx == rq->ring->pidx) { 1539 device_printf(sc->dev, 1540 "oce_rx_mbuf_chain: Invalid RX completion - Queue is empty\n"); 1541 return; 1542 } 1543 pd = &rq->pckts[rq->ring->cidx]; 1544 1545 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1546 bus_dmamap_unload(rq->tag, pd->map); 1547 RING_GET(rq->ring, 1); 1548 rq->pending--; 1549 1550 frag_len = (len > rq->cfg.frag_size) ? rq->cfg.frag_size : len; 1551 pd->mbuf->m_len = frag_len; 1552 1553 if (tail != NULL) { 1554 /* additional fragments */ 1555 pd->mbuf->m_flags &= ~M_PKTHDR; 1556 tail->m_next = pd->mbuf; 1557 if(rq->islro) 1558 tail->m_nextpkt = NULL; 1559 tail = pd->mbuf; 1560 } else { 1561 /* first fragment, fill out much of the packet header */ 1562 pd->mbuf->m_pkthdr.len = len; 1563 if(rq->islro) 1564 pd->mbuf->m_nextpkt = NULL; 1565 pd->mbuf->m_pkthdr.csum_flags = 0; 1566 if (IF_CSUM_ENABLED(sc)) { 1567 if (cqe_info->l4_cksum_pass) { 1568 if(!cqe_info->ipv6_frame) { /* IPV4 */ 1569 pd->mbuf->m_pkthdr.csum_flags |= 1570 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1571 }else { /* IPV6 frame */ 1572 if(rq->islro) { 1573 pd->mbuf->m_pkthdr.csum_flags |= 1574 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1575 } 1576 } 1577 pd->mbuf->m_pkthdr.csum_data = 0xffff; 1578 } 1579 if (cqe_info->ip_cksum_pass) { 1580 pd->mbuf->m_pkthdr.csum_flags |= 1581 (CSUM_IP_CHECKED|CSUM_IP_VALID); 1582 } 1583 } 1584 *m = tail = pd->mbuf; 1585 } 1586 pd->mbuf = NULL; 1587 len -= frag_len; 1588 } 1589 1590 return; 1591 } 1592 1593 static void 1594 oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2) 1595 { 1596 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1597 struct nic_hwlro_cqe_part1 *cqe1 = NULL; 1598 struct mbuf *m = NULL; 1599 struct oce_common_cqe_info cq_info; 1600 1601 /* parse cqe */ 1602 if(cqe2 == NULL) { 1603 cq_info.pkt_size = cqe->pkt_size; 1604 cq_info.vtag = cqe->vlan_tag; 1605 cq_info.l4_cksum_pass = cqe->l4_cksum_pass; 1606 cq_info.ip_cksum_pass = cqe->ip_cksum_pass; 1607 cq_info.ipv6_frame = cqe->ipv6_frame; 1608 cq_info.vtp = cqe->vtp; 1609 cq_info.qnq = cqe->qnq; 1610 }else { 1611 cqe1 = (struct nic_hwlro_cqe_part1 *)cqe; 1612 cq_info.pkt_size = cqe2->coalesced_size; 1613 cq_info.vtag = cqe2->vlan_tag; 1614 cq_info.l4_cksum_pass = cqe2->l4_cksum_pass; 1615 cq_info.ip_cksum_pass = cqe2->ip_cksum_pass; 1616 cq_info.ipv6_frame = cqe2->ipv6_frame; 1617 cq_info.vtp = cqe2->vtp; 1618 cq_info.qnq = cqe1->qnq; 1619 } 1620 1621 cq_info.vtag = BSWAP_16(cq_info.vtag); 1622 1623 cq_info.num_frags = cq_info.pkt_size / rq->cfg.frag_size; 1624 if(cq_info.pkt_size % rq->cfg.frag_size) 1625 cq_info.num_frags++; 1626 1627 oce_rx_mbuf_chain(rq, &cq_info, &m); 1628 1629 if (m) { 1630 if(cqe2) { 1631 //assert(cqe2->valid != 0); 1632 1633 //assert(cqe2->cqe_type != 2); 1634 oce_correct_header(m, cqe1, cqe2); 1635 } 1636 1637 m->m_pkthdr.rcvif = sc->ifp; 1638 #if __FreeBSD_version >= 800000 1639 if (rq->queue_index) 1640 m->m_pkthdr.flowid = (rq->queue_index - 1); 1641 else 1642 m->m_pkthdr.flowid = rq->queue_index; 1643 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 1644 #endif 1645 /* This deternies if vlan tag is Valid */ 1646 if (cq_info.vtp) { 1647 if (sc->function_mode & FNM_FLEX10_MODE) { 1648 /* FLEX10. If QnQ is not set, neglect VLAN */ 1649 if (cq_info.qnq) { 1650 m->m_pkthdr.ether_vtag = cq_info.vtag; 1651 m->m_flags |= M_VLANTAG; 1652 } 1653 } else if (sc->pvid != (cq_info.vtag & VLAN_VID_MASK)) { 1654 /* In UMC mode generally pvid will be striped by 1655 hw. But in some cases we have seen it comes 1656 with pvid. So if pvid == vlan, neglect vlan. 1657 */ 1658 m->m_pkthdr.ether_vtag = cq_info.vtag; 1659 m->m_flags |= M_VLANTAG; 1660 } 1661 } 1662 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 1663 1664 (*sc->ifp->if_input) (sc->ifp, m); 1665 1666 /* Update rx stats per queue */ 1667 rq->rx_stats.rx_pkts++; 1668 rq->rx_stats.rx_bytes += cq_info.pkt_size; 1669 rq->rx_stats.rx_frags += cq_info.num_frags; 1670 rq->rx_stats.rx_ucast_pkts++; 1671 } 1672 return; 1673 } 1674 1675 static void 1676 oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe) 1677 { 1678 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1679 int len; 1680 struct mbuf *m = NULL; 1681 struct oce_common_cqe_info cq_info; 1682 uint16_t vtag = 0; 1683 1684 /* Is it a flush compl that has no data */ 1685 if(!cqe->u0.s.num_fragments) 1686 goto exit; 1687 1688 len = cqe->u0.s.pkt_size; 1689 if (!len) { 1690 /*partial DMA workaround for Lancer*/ 1691 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); 1692 goto exit; 1693 } 1694 1695 if (!oce_cqe_portid_valid(sc, cqe)) { 1696 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); 1697 goto exit; 1698 } 1699 1700 /* Get vlan_tag value */ 1701 if(IS_BE(sc) || IS_SH(sc)) 1702 vtag = BSWAP_16(cqe->u0.s.vlan_tag); 1703 else 1704 vtag = cqe->u0.s.vlan_tag; 1705 1706 cq_info.l4_cksum_pass = cqe->u0.s.l4_cksum_pass; 1707 cq_info.ip_cksum_pass = cqe->u0.s.ip_cksum_pass; 1708 cq_info.ipv6_frame = cqe->u0.s.ip_ver; 1709 cq_info.num_frags = cqe->u0.s.num_fragments; 1710 cq_info.pkt_size = cqe->u0.s.pkt_size; 1711 1712 oce_rx_mbuf_chain(rq, &cq_info, &m); 1713 1714 if (m) { 1715 m->m_pkthdr.rcvif = sc->ifp; 1716 #if __FreeBSD_version >= 800000 1717 if (rq->queue_index) 1718 m->m_pkthdr.flowid = (rq->queue_index - 1); 1719 else 1720 m->m_pkthdr.flowid = rq->queue_index; 1721 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 1722 #endif 1723 /* This deternies if vlan tag is Valid */ 1724 if (oce_cqe_vtp_valid(sc, cqe)) { 1725 if (sc->function_mode & FNM_FLEX10_MODE) { 1726 /* FLEX10. If QnQ is not set, neglect VLAN */ 1727 if (cqe->u0.s.qnq) { 1728 m->m_pkthdr.ether_vtag = vtag; 1729 m->m_flags |= M_VLANTAG; 1730 } 1731 } else if (sc->pvid != (vtag & VLAN_VID_MASK)) { 1732 /* In UMC mode generally pvid will be striped by 1733 hw. But in some cases we have seen it comes 1734 with pvid. So if pvid == vlan, neglect vlan. 1735 */ 1736 m->m_pkthdr.ether_vtag = vtag; 1737 m->m_flags |= M_VLANTAG; 1738 } 1739 } 1740 1741 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 1742 #if defined(INET6) || defined(INET) 1743 /* Try to queue to LRO */ 1744 if (IF_LRO_ENABLED(sc) && 1745 (cqe->u0.s.ip_cksum_pass) && 1746 (cqe->u0.s.l4_cksum_pass) && 1747 (!cqe->u0.s.ip_ver) && 1748 (rq->lro.lro_cnt != 0)) { 1749 1750 if (tcp_lro_rx(&rq->lro, m, 0) == 0) { 1751 rq->lro_pkts_queued ++; 1752 goto post_done; 1753 } 1754 /* If LRO posting fails then try to post to STACK */ 1755 } 1756 #endif 1757 1758 (*sc->ifp->if_input) (sc->ifp, m); 1759 #if defined(INET6) || defined(INET) 1760 post_done: 1761 #endif 1762 /* Update rx stats per queue */ 1763 rq->rx_stats.rx_pkts++; 1764 rq->rx_stats.rx_bytes += cqe->u0.s.pkt_size; 1765 rq->rx_stats.rx_frags += cqe->u0.s.num_fragments; 1766 if (cqe->u0.s.pkt_type == OCE_MULTICAST_PACKET) 1767 rq->rx_stats.rx_mcast_pkts++; 1768 if (cqe->u0.s.pkt_type == OCE_UNICAST_PACKET) 1769 rq->rx_stats.rx_ucast_pkts++; 1770 } 1771 exit: 1772 return; 1773 } 1774 1775 1776 void 1777 oce_discard_rx_comp(struct oce_rq *rq, int num_frags) 1778 { 1779 uint32_t i = 0; 1780 struct oce_packet_desc *pd; 1781 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1782 1783 for (i = 0; i < num_frags; i++) { 1784 if (rq->ring->cidx == rq->ring->pidx) { 1785 device_printf(sc->dev, 1786 "oce_discard_rx_comp: Invalid RX completion - Queue is empty\n"); 1787 return; 1788 } 1789 pd = &rq->pckts[rq->ring->cidx]; 1790 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1791 bus_dmamap_unload(rq->tag, pd->map); 1792 if (pd->mbuf != NULL) { 1793 m_freem(pd->mbuf); 1794 pd->mbuf = NULL; 1795 } 1796 1797 RING_GET(rq->ring, 1); 1798 rq->pending--; 1799 } 1800 } 1801 1802 1803 static int 1804 oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe) 1805 { 1806 struct oce_nic_rx_cqe_v1 *cqe_v1; 1807 int vtp = 0; 1808 1809 if (sc->be3_native) { 1810 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe; 1811 vtp = cqe_v1->u0.s.vlan_tag_present; 1812 } else 1813 vtp = cqe->u0.s.vlan_tag_present; 1814 1815 return vtp; 1816 1817 } 1818 1819 1820 static int 1821 oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe) 1822 { 1823 struct oce_nic_rx_cqe_v1 *cqe_v1; 1824 int port_id = 0; 1825 1826 if (sc->be3_native && (IS_BE(sc) || IS_SH(sc))) { 1827 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe; 1828 port_id = cqe_v1->u0.s.port; 1829 if (sc->port_id != port_id) 1830 return 0; 1831 } else 1832 ;/* For BE3 legacy and Lancer this is dummy */ 1833 1834 return 1; 1835 1836 } 1837 1838 #if defined(INET6) || defined(INET) 1839 void 1840 oce_rx_flush_lro(struct oce_rq *rq) 1841 { 1842 struct lro_ctrl *lro = &rq->lro; 1843 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1844 1845 if (!IF_LRO_ENABLED(sc)) 1846 return; 1847 1848 tcp_lro_flush_all(lro); 1849 rq->lro_pkts_queued = 0; 1850 1851 return; 1852 } 1853 1854 1855 static int 1856 oce_init_lro(POCE_SOFTC sc) 1857 { 1858 struct lro_ctrl *lro = NULL; 1859 int i = 0, rc = 0; 1860 1861 for (i = 0; i < sc->nrqs; i++) { 1862 lro = &sc->rq[i]->lro; 1863 rc = tcp_lro_init(lro); 1864 if (rc != 0) { 1865 device_printf(sc->dev, "LRO init failed\n"); 1866 return rc; 1867 } 1868 lro->ifp = sc->ifp; 1869 } 1870 1871 return rc; 1872 } 1873 1874 1875 void 1876 oce_free_lro(POCE_SOFTC sc) 1877 { 1878 struct lro_ctrl *lro = NULL; 1879 int i = 0; 1880 1881 for (i = 0; i < sc->nrqs; i++) { 1882 lro = &sc->rq[i]->lro; 1883 if (lro) 1884 tcp_lro_free(lro); 1885 } 1886 } 1887 #endif 1888 1889 int 1890 oce_alloc_rx_bufs(struct oce_rq *rq, int count) 1891 { 1892 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1893 int i, in, rc; 1894 struct oce_packet_desc *pd; 1895 bus_dma_segment_t segs[6]; 1896 int nsegs, added = 0; 1897 struct oce_nic_rqe *rqe; 1898 pd_rxulp_db_t rxdb_reg; 1899 uint32_t val = 0; 1900 uint32_t oce_max_rq_posts = 64; 1901 1902 bzero(&rxdb_reg, sizeof(pd_rxulp_db_t)); 1903 for (i = 0; i < count; i++) { 1904 in = (rq->ring->pidx + 1) % OCE_RQ_PACKET_ARRAY_SIZE; 1905 1906 pd = &rq->pckts[rq->ring->pidx]; 1907 pd->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, oce_rq_buf_size); 1908 if (pd->mbuf == NULL) { 1909 device_printf(sc->dev, "mbuf allocation failed, size = %d\n",oce_rq_buf_size); 1910 break; 1911 } 1912 pd->mbuf->m_nextpkt = NULL; 1913 1914 pd->mbuf->m_len = pd->mbuf->m_pkthdr.len = rq->cfg.frag_size; 1915 1916 rc = bus_dmamap_load_mbuf_sg(rq->tag, 1917 pd->map, 1918 pd->mbuf, 1919 segs, &nsegs, BUS_DMA_NOWAIT); 1920 if (rc) { 1921 m_free(pd->mbuf); 1922 device_printf(sc->dev, "bus_dmamap_load_mbuf_sg failed rc = %d\n", rc); 1923 break; 1924 } 1925 1926 if (nsegs != 1) { 1927 i--; 1928 continue; 1929 } 1930 1931 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_PREREAD); 1932 1933 rqe = RING_GET_PRODUCER_ITEM_VA(rq->ring, struct oce_nic_rqe); 1934 rqe->u0.s.frag_pa_hi = ADDR_HI(segs[0].ds_addr); 1935 rqe->u0.s.frag_pa_lo = ADDR_LO(segs[0].ds_addr); 1936 DW_SWAP(u32ptr(rqe), sizeof(struct oce_nic_rqe)); 1937 RING_PUT(rq->ring, 1); 1938 added++; 1939 rq->pending++; 1940 } 1941 oce_max_rq_posts = sc->enable_hwlro ? OCE_HWLRO_MAX_RQ_POSTS : OCE_MAX_RQ_POSTS; 1942 if (added != 0) { 1943 for (i = added / oce_max_rq_posts; i > 0; i--) { 1944 rxdb_reg.bits.num_posted = oce_max_rq_posts; 1945 rxdb_reg.bits.qid = rq->rq_id; 1946 if(rq->islro) { 1947 val |= rq->rq_id & DB_LRO_RQ_ID_MASK; 1948 val |= oce_max_rq_posts << 16; 1949 OCE_WRITE_REG32(sc, db, DB_OFFSET, val); 1950 }else { 1951 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); 1952 } 1953 added -= oce_max_rq_posts; 1954 } 1955 if (added > 0) { 1956 rxdb_reg.bits.qid = rq->rq_id; 1957 rxdb_reg.bits.num_posted = added; 1958 if(rq->islro) { 1959 val |= rq->rq_id & DB_LRO_RQ_ID_MASK; 1960 val |= added << 16; 1961 OCE_WRITE_REG32(sc, db, DB_OFFSET, val); 1962 }else { 1963 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); 1964 } 1965 } 1966 } 1967 1968 return 0; 1969 } 1970 1971 static void 1972 oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq) 1973 { 1974 if (num_cqes) { 1975 oce_arm_cq(sc, rq->cq->cq_id, num_cqes, FALSE); 1976 if(!sc->enable_hwlro) { 1977 if((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) > 1) 1978 oce_alloc_rx_bufs(rq, ((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) - 1)); 1979 }else { 1980 if ((OCE_RQ_PACKET_ARRAY_SIZE -1 - rq->pending) > 64) 1981 oce_alloc_rx_bufs(rq, 64); 1982 } 1983 } 1984 1985 return; 1986 } 1987 1988 uint16_t 1989 oce_rq_handler_lro(void *arg) 1990 { 1991 struct oce_rq *rq = (struct oce_rq *)arg; 1992 struct oce_cq *cq = rq->cq; 1993 POCE_SOFTC sc = rq->parent; 1994 struct nic_hwlro_singleton_cqe *cqe; 1995 struct nic_hwlro_cqe_part2 *cqe2; 1996 int num_cqes = 0; 1997 1998 LOCK(&rq->rx_lock); 1999 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2000 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); 2001 while (cqe->valid) { 2002 if(cqe->cqe_type == 0) { /* singleton cqe */ 2003 /* we should not get singleton cqe after cqe1 on same rq */ 2004 if(rq->cqe_firstpart != NULL) { 2005 device_printf(sc->dev, "Got singleton cqe after cqe1 \n"); 2006 goto exit_rq_handler_lro; 2007 } 2008 if(cqe->error != 0) { 2009 rq->rx_stats.rxcp_err++; 2010 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2011 } 2012 oce_rx_lro(rq, cqe, NULL); 2013 rq->rx_stats.rx_compl++; 2014 cqe->valid = 0; 2015 RING_GET(cq->ring, 1); 2016 num_cqes++; 2017 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2018 break; 2019 }else if(cqe->cqe_type == 0x1) { /* first part */ 2020 /* we should not get cqe1 after cqe1 on same rq */ 2021 if(rq->cqe_firstpart != NULL) { 2022 device_printf(sc->dev, "Got cqe1 after cqe1 \n"); 2023 goto exit_rq_handler_lro; 2024 } 2025 rq->cqe_firstpart = (struct nic_hwlro_cqe_part1 *)cqe; 2026 RING_GET(cq->ring, 1); 2027 }else if(cqe->cqe_type == 0x2) { /* second part */ 2028 cqe2 = (struct nic_hwlro_cqe_part2 *)cqe; 2029 if(cqe2->error != 0) { 2030 rq->rx_stats.rxcp_err++; 2031 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2032 } 2033 /* We should not get cqe2 without cqe1 */ 2034 if(rq->cqe_firstpart == NULL) { 2035 device_printf(sc->dev, "Got cqe2 without cqe1 \n"); 2036 goto exit_rq_handler_lro; 2037 } 2038 oce_rx_lro(rq, (struct nic_hwlro_singleton_cqe *)rq->cqe_firstpart, cqe2); 2039 2040 rq->rx_stats.rx_compl++; 2041 rq->cqe_firstpart->valid = 0; 2042 cqe2->valid = 0; 2043 rq->cqe_firstpart = NULL; 2044 2045 RING_GET(cq->ring, 1); 2046 num_cqes += 2; 2047 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2048 break; 2049 } 2050 2051 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2052 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); 2053 } 2054 oce_check_rx_bufs(sc, num_cqes, rq); 2055 exit_rq_handler_lro: 2056 UNLOCK(&rq->rx_lock); 2057 return 0; 2058 } 2059 2060 /* Handle the Completion Queue for receive */ 2061 uint16_t 2062 oce_rq_handler(void *arg) 2063 { 2064 struct oce_rq *rq = (struct oce_rq *)arg; 2065 struct oce_cq *cq = rq->cq; 2066 POCE_SOFTC sc = rq->parent; 2067 struct oce_nic_rx_cqe *cqe; 2068 int num_cqes = 0; 2069 2070 if(rq->islro) { 2071 oce_rq_handler_lro(arg); 2072 return 0; 2073 } 2074 LOCK(&rq->rx_lock); 2075 bus_dmamap_sync(cq->ring->dma.tag, 2076 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2077 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); 2078 while (cqe->u0.dw[2]) { 2079 DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe)); 2080 2081 if (cqe->u0.s.error == 0) { 2082 oce_rx(rq, cqe); 2083 } else { 2084 rq->rx_stats.rxcp_err++; 2085 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2086 /* Post L3/L4 errors to stack.*/ 2087 oce_rx(rq, cqe); 2088 } 2089 rq->rx_stats.rx_compl++; 2090 cqe->u0.dw[2] = 0; 2091 2092 #if defined(INET6) || defined(INET) 2093 if (IF_LRO_ENABLED(sc) && rq->lro_pkts_queued >= 16) { 2094 oce_rx_flush_lro(rq); 2095 } 2096 #endif 2097 2098 RING_GET(cq->ring, 1); 2099 bus_dmamap_sync(cq->ring->dma.tag, 2100 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2101 cqe = 2102 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); 2103 num_cqes++; 2104 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2105 break; 2106 } 2107 2108 #if defined(INET6) || defined(INET) 2109 if (IF_LRO_ENABLED(sc)) 2110 oce_rx_flush_lro(rq); 2111 #endif 2112 2113 oce_check_rx_bufs(sc, num_cqes, rq); 2114 UNLOCK(&rq->rx_lock); 2115 return 0; 2116 2117 } 2118 2119 2120 2121 2122 /***************************************************************************** 2123 * Helper function prototypes in this file * 2124 *****************************************************************************/ 2125 2126 static int 2127 oce_attach_ifp(POCE_SOFTC sc) 2128 { 2129 2130 sc->ifp = if_alloc(IFT_ETHER); 2131 if (!sc->ifp) 2132 return ENOMEM; 2133 2134 ifmedia_init(&sc->media, IFM_IMASK, oce_media_change, oce_media_status); 2135 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2136 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 2137 2138 sc->ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST; 2139 sc->ifp->if_ioctl = oce_ioctl; 2140 sc->ifp->if_start = oce_start; 2141 sc->ifp->if_init = oce_init; 2142 sc->ifp->if_mtu = ETHERMTU; 2143 sc->ifp->if_softc = sc; 2144 #if __FreeBSD_version >= 800000 2145 sc->ifp->if_transmit = oce_multiq_start; 2146 sc->ifp->if_qflush = oce_multiq_flush; 2147 #endif 2148 2149 if_initname(sc->ifp, 2150 device_get_name(sc->dev), device_get_unit(sc->dev)); 2151 2152 sc->ifp->if_snd.ifq_drv_maxlen = OCE_MAX_TX_DESC - 1; 2153 IFQ_SET_MAXLEN(&sc->ifp->if_snd, sc->ifp->if_snd.ifq_drv_maxlen); 2154 IFQ_SET_READY(&sc->ifp->if_snd); 2155 2156 sc->ifp->if_hwassist = OCE_IF_HWASSIST; 2157 sc->ifp->if_hwassist |= CSUM_TSO; 2158 sc->ifp->if_hwassist |= (CSUM_IP | CSUM_TCP | CSUM_UDP); 2159 2160 sc->ifp->if_capabilities = OCE_IF_CAPABILITIES; 2161 sc->ifp->if_capabilities |= IFCAP_HWCSUM; 2162 sc->ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 2163 2164 #if defined(INET6) || defined(INET) 2165 sc->ifp->if_capabilities |= IFCAP_TSO; 2166 sc->ifp->if_capabilities |= IFCAP_LRO; 2167 sc->ifp->if_capabilities |= IFCAP_VLAN_HWTSO; 2168 #endif 2169 2170 sc->ifp->if_capenable = sc->ifp->if_capabilities; 2171 sc->ifp->if_baudrate = IF_Gbps(10); 2172 2173 #if __FreeBSD_version >= 1000000 2174 sc->ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 2175 sc->ifp->if_hw_tsomaxsegcount = OCE_MAX_TX_ELEMENTS; 2176 sc->ifp->if_hw_tsomaxsegsize = 4096; 2177 #endif 2178 2179 ether_ifattach(sc->ifp, sc->macaddr.mac_addr); 2180 2181 return 0; 2182 } 2183 2184 2185 static void 2186 oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag) 2187 { 2188 POCE_SOFTC sc = ifp->if_softc; 2189 2190 if (ifp->if_softc != arg) 2191 return; 2192 if ((vtag == 0) || (vtag > 4095)) 2193 return; 2194 2195 sc->vlan_tag[vtag] = 1; 2196 sc->vlans_added++; 2197 if (sc->vlans_added <= (sc->max_vlans + 1)) 2198 oce_vid_config(sc); 2199 } 2200 2201 2202 static void 2203 oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag) 2204 { 2205 POCE_SOFTC sc = ifp->if_softc; 2206 2207 if (ifp->if_softc != arg) 2208 return; 2209 if ((vtag == 0) || (vtag > 4095)) 2210 return; 2211 2212 sc->vlan_tag[vtag] = 0; 2213 sc->vlans_added--; 2214 oce_vid_config(sc); 2215 } 2216 2217 2218 /* 2219 * A max of 64 vlans can be configured in BE. If the user configures 2220 * more, place the card in vlan promiscuous mode. 2221 */ 2222 static int 2223 oce_vid_config(POCE_SOFTC sc) 2224 { 2225 struct normal_vlan vtags[MAX_VLANFILTER_SIZE]; 2226 uint16_t ntags = 0, i; 2227 int status = 0; 2228 2229 if ((sc->vlans_added <= MAX_VLANFILTER_SIZE) && 2230 (sc->ifp->if_capenable & IFCAP_VLAN_HWFILTER)) { 2231 for (i = 0; i < MAX_VLANS; i++) { 2232 if (sc->vlan_tag[i]) { 2233 vtags[ntags].vtag = i; 2234 ntags++; 2235 } 2236 } 2237 if (ntags) 2238 status = oce_config_vlan(sc, (uint8_t) sc->if_id, 2239 vtags, ntags, 1, 0); 2240 } else 2241 status = oce_config_vlan(sc, (uint8_t) sc->if_id, 2242 NULL, 0, 1, 1); 2243 return status; 2244 } 2245 2246 2247 static void 2248 oce_mac_addr_set(POCE_SOFTC sc) 2249 { 2250 uint32_t old_pmac_id = sc->pmac_id; 2251 int status = 0; 2252 2253 2254 status = bcmp((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr, 2255 sc->macaddr.size_of_struct); 2256 if (!status) 2257 return; 2258 2259 status = oce_mbox_macaddr_add(sc, (uint8_t *)(IF_LLADDR(sc->ifp)), 2260 sc->if_id, &sc->pmac_id); 2261 if (!status) { 2262 status = oce_mbox_macaddr_del(sc, sc->if_id, old_pmac_id); 2263 bcopy((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr, 2264 sc->macaddr.size_of_struct); 2265 } 2266 if (status) 2267 device_printf(sc->dev, "Failed update macaddress\n"); 2268 2269 } 2270 2271 2272 static int 2273 oce_handle_passthrough(struct ifnet *ifp, caddr_t data) 2274 { 2275 POCE_SOFTC sc = ifp->if_softc; 2276 struct ifreq *ifr = (struct ifreq *)data; 2277 int rc = ENXIO; 2278 char cookie[32] = {0}; 2279 void *priv_data = (void *)ifr->ifr_data; 2280 void *ioctl_ptr; 2281 uint32_t req_size; 2282 struct mbx_hdr req; 2283 OCE_DMA_MEM dma_mem; 2284 struct mbx_common_get_cntl_attr *fw_cmd; 2285 2286 if (copyin(priv_data, cookie, strlen(IOCTL_COOKIE))) 2287 return EFAULT; 2288 2289 if (memcmp(cookie, IOCTL_COOKIE, strlen(IOCTL_COOKIE))) 2290 return EINVAL; 2291 2292 ioctl_ptr = (char *)priv_data + strlen(IOCTL_COOKIE); 2293 if (copyin(ioctl_ptr, &req, sizeof(struct mbx_hdr))) 2294 return EFAULT; 2295 2296 req_size = le32toh(req.u0.req.request_length); 2297 if (req_size > 65536) 2298 return EINVAL; 2299 2300 req_size += sizeof(struct mbx_hdr); 2301 rc = oce_dma_alloc(sc, req_size, &dma_mem, 0); 2302 if (rc) 2303 return ENOMEM; 2304 2305 if (copyin(ioctl_ptr, OCE_DMAPTR(&dma_mem,char), req_size)) { 2306 rc = EFAULT; 2307 goto dma_free; 2308 } 2309 2310 rc = oce_pass_through_mbox(sc, &dma_mem, req_size); 2311 if (rc) { 2312 rc = EIO; 2313 goto dma_free; 2314 } 2315 2316 if (copyout(OCE_DMAPTR(&dma_mem,char), ioctl_ptr, req_size)) 2317 rc = EFAULT; 2318 2319 /* 2320 firmware is filling all the attributes for this ioctl except 2321 the driver version..so fill it 2322 */ 2323 if(req.u0.rsp.opcode == OPCODE_COMMON_GET_CNTL_ATTRIBUTES) { 2324 fw_cmd = (struct mbx_common_get_cntl_attr *) ioctl_ptr; 2325 strncpy(fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str, 2326 COMPONENT_REVISION, strlen(COMPONENT_REVISION)); 2327 } 2328 2329 dma_free: 2330 oce_dma_free(sc, &dma_mem); 2331 return rc; 2332 2333 } 2334 2335 static void 2336 oce_eqd_set_periodic(POCE_SOFTC sc) 2337 { 2338 struct oce_set_eqd set_eqd[OCE_MAX_EQ]; 2339 struct oce_aic_obj *aic; 2340 struct oce_eq *eqo; 2341 uint64_t now = 0, delta; 2342 int eqd, i, num = 0; 2343 uint32_t tx_reqs = 0, rxpkts = 0, pps; 2344 struct oce_wq *wq; 2345 struct oce_rq *rq; 2346 2347 #define ticks_to_msecs(t) (1000 * (t) / hz) 2348 2349 for (i = 0 ; i < sc->neqs; i++) { 2350 eqo = sc->eq[i]; 2351 aic = &sc->aic_obj[i]; 2352 /* When setting the static eq delay from the user space */ 2353 if (!aic->enable) { 2354 if (aic->ticks) 2355 aic->ticks = 0; 2356 eqd = aic->et_eqd; 2357 goto modify_eqd; 2358 } 2359 2360 rq = sc->rq[i]; 2361 rxpkts = rq->rx_stats.rx_pkts; 2362 wq = sc->wq[i]; 2363 tx_reqs = wq->tx_stats.tx_reqs; 2364 now = ticks; 2365 2366 if (!aic->ticks || now < aic->ticks || 2367 rxpkts < aic->prev_rxpkts || tx_reqs < aic->prev_txreqs) { 2368 aic->prev_rxpkts = rxpkts; 2369 aic->prev_txreqs = tx_reqs; 2370 aic->ticks = now; 2371 continue; 2372 } 2373 2374 delta = ticks_to_msecs(now - aic->ticks); 2375 2376 pps = (((uint32_t)(rxpkts - aic->prev_rxpkts) * 1000) / delta) + 2377 (((uint32_t)(tx_reqs - aic->prev_txreqs) * 1000) / delta); 2378 eqd = (pps / 15000) << 2; 2379 if (eqd < 8) 2380 eqd = 0; 2381 2382 /* Make sure that the eq delay is in the known range */ 2383 eqd = min(eqd, aic->max_eqd); 2384 eqd = max(eqd, aic->min_eqd); 2385 2386 aic->prev_rxpkts = rxpkts; 2387 aic->prev_txreqs = tx_reqs; 2388 aic->ticks = now; 2389 2390 modify_eqd: 2391 if (eqd != aic->cur_eqd) { 2392 set_eqd[num].delay_multiplier = (eqd * 65)/100; 2393 set_eqd[num].eq_id = eqo->eq_id; 2394 aic->cur_eqd = eqd; 2395 num++; 2396 } 2397 } 2398 2399 /* Is there atleast one eq that needs to be modified? */ 2400 for(i = 0; i < num; i += 8) { 2401 if((num - i) >=8 ) 2402 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], 8); 2403 else 2404 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], (num - i)); 2405 } 2406 2407 } 2408 2409 static void oce_detect_hw_error(POCE_SOFTC sc) 2410 { 2411 2412 uint32_t ue_low = 0, ue_high = 0, ue_low_mask = 0, ue_high_mask = 0; 2413 uint32_t sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0; 2414 uint32_t i; 2415 2416 if (sc->hw_error) 2417 return; 2418 2419 if (IS_XE201(sc)) { 2420 sliport_status = OCE_READ_REG32(sc, db, SLIPORT_STATUS_OFFSET); 2421 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2422 sliport_err1 = OCE_READ_REG32(sc, db, SLIPORT_ERROR1_OFFSET); 2423 sliport_err2 = OCE_READ_REG32(sc, db, SLIPORT_ERROR2_OFFSET); 2424 } 2425 } else { 2426 ue_low = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW); 2427 ue_high = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HIGH); 2428 ue_low_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW_MASK); 2429 ue_high_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HI_MASK); 2430 2431 ue_low = (ue_low & ~ue_low_mask); 2432 ue_high = (ue_high & ~ue_high_mask); 2433 } 2434 2435 /* On certain platforms BE hardware can indicate spurious UEs. 2436 * Allow the h/w to stop working completely in case of a real UE. 2437 * Hence not setting the hw_error for UE detection. 2438 */ 2439 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2440 sc->hw_error = TRUE; 2441 device_printf(sc->dev, "Error detected in the card\n"); 2442 } 2443 2444 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2445 device_printf(sc->dev, 2446 "ERR: sliport status 0x%x\n", sliport_status); 2447 device_printf(sc->dev, 2448 "ERR: sliport error1 0x%x\n", sliport_err1); 2449 device_printf(sc->dev, 2450 "ERR: sliport error2 0x%x\n", sliport_err2); 2451 } 2452 2453 if (ue_low) { 2454 for (i = 0; ue_low; ue_low >>= 1, i++) { 2455 if (ue_low & 1) 2456 device_printf(sc->dev, "UE: %s bit set\n", 2457 ue_status_low_desc[i]); 2458 } 2459 } 2460 2461 if (ue_high) { 2462 for (i = 0; ue_high; ue_high >>= 1, i++) { 2463 if (ue_high & 1) 2464 device_printf(sc->dev, "UE: %s bit set\n", 2465 ue_status_hi_desc[i]); 2466 } 2467 } 2468 2469 } 2470 2471 2472 static void 2473 oce_local_timer(void *arg) 2474 { 2475 POCE_SOFTC sc = arg; 2476 int i = 0; 2477 2478 oce_detect_hw_error(sc); 2479 oce_refresh_nic_stats(sc); 2480 oce_refresh_queue_stats(sc); 2481 oce_mac_addr_set(sc); 2482 2483 /* TX Watch Dog*/ 2484 for (i = 0; i < sc->nwqs; i++) 2485 oce_tx_restart(sc, sc->wq[i]); 2486 2487 /* calculate and set the eq delay for optimal interrupt rate */ 2488 if (IS_BE(sc) || IS_SH(sc)) 2489 oce_eqd_set_periodic(sc); 2490 2491 callout_reset(&sc->timer, hz, oce_local_timer, sc); 2492 } 2493 2494 static void 2495 oce_tx_compl_clean(POCE_SOFTC sc) 2496 { 2497 struct oce_wq *wq; 2498 int i = 0, timeo = 0, num_wqes = 0; 2499 int pending_txqs = sc->nwqs; 2500 2501 /* Stop polling for compls when HW has been silent for 10ms or 2502 * hw_error or no outstanding completions expected 2503 */ 2504 do { 2505 pending_txqs = sc->nwqs; 2506 2507 for_all_wq_queues(sc, wq, i) { 2508 num_wqes = oce_wq_handler(wq); 2509 2510 if(num_wqes) 2511 timeo = 0; 2512 2513 if(!wq->ring->num_used) 2514 pending_txqs--; 2515 } 2516 2517 if (pending_txqs == 0 || ++timeo > 10 || sc->hw_error) 2518 break; 2519 2520 DELAY(1000); 2521 } while (TRUE); 2522 2523 for_all_wq_queues(sc, wq, i) { 2524 while(wq->ring->num_used) { 2525 LOCK(&wq->tx_compl_lock); 2526 oce_process_tx_completion(wq); 2527 UNLOCK(&wq->tx_compl_lock); 2528 } 2529 } 2530 2531 } 2532 2533 /* NOTE : This should only be called holding 2534 * DEVICE_LOCK. 2535 */ 2536 static void 2537 oce_if_deactivate(POCE_SOFTC sc) 2538 { 2539 int i; 2540 struct oce_rq *rq; 2541 struct oce_wq *wq; 2542 struct oce_eq *eq; 2543 2544 sc->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 2545 2546 oce_tx_compl_clean(sc); 2547 2548 /* Stop intrs and finish any bottom halves pending */ 2549 oce_hw_intr_disable(sc); 2550 2551 /* Since taskqueue_drain takes a Gaint Lock, We should not acquire 2552 any other lock. So unlock device lock and require after 2553 completing taskqueue_drain. 2554 */ 2555 UNLOCK(&sc->dev_lock); 2556 for (i = 0; i < sc->intr_count; i++) { 2557 if (sc->intrs[i].tq != NULL) { 2558 taskqueue_drain(sc->intrs[i].tq, &sc->intrs[i].task); 2559 } 2560 } 2561 LOCK(&sc->dev_lock); 2562 2563 /* Delete RX queue in card with flush param */ 2564 oce_stop_rx(sc); 2565 2566 /* Invalidate any pending cq and eq entries*/ 2567 for_all_evnt_queues(sc, eq, i) 2568 oce_drain_eq(eq); 2569 for_all_rq_queues(sc, rq, i) 2570 oce_drain_rq_cq(rq); 2571 for_all_wq_queues(sc, wq, i) 2572 oce_drain_wq_cq(wq); 2573 2574 /* But still we need to get MCC aync events. 2575 So enable intrs and also arm first EQ 2576 */ 2577 oce_hw_intr_enable(sc); 2578 oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE); 2579 2580 DELAY(10); 2581 } 2582 2583 2584 static void 2585 oce_if_activate(POCE_SOFTC sc) 2586 { 2587 struct oce_eq *eq; 2588 struct oce_rq *rq; 2589 struct oce_wq *wq; 2590 int i, rc = 0; 2591 2592 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; 2593 2594 oce_hw_intr_disable(sc); 2595 2596 oce_start_rx(sc); 2597 2598 for_all_rq_queues(sc, rq, i) { 2599 rc = oce_start_rq(rq); 2600 if (rc) 2601 device_printf(sc->dev, "Unable to start RX\n"); 2602 } 2603 2604 for_all_wq_queues(sc, wq, i) { 2605 rc = oce_start_wq(wq); 2606 if (rc) 2607 device_printf(sc->dev, "Unable to start TX\n"); 2608 } 2609 2610 2611 for_all_evnt_queues(sc, eq, i) 2612 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); 2613 2614 oce_hw_intr_enable(sc); 2615 2616 } 2617 2618 static void 2619 process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe) 2620 { 2621 /* Update Link status */ 2622 if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) == 2623 ASYNC_EVENT_LINK_UP) { 2624 sc->link_status = ASYNC_EVENT_LINK_UP; 2625 if_link_state_change(sc->ifp, LINK_STATE_UP); 2626 } else { 2627 sc->link_status = ASYNC_EVENT_LINK_DOWN; 2628 if_link_state_change(sc->ifp, LINK_STATE_DOWN); 2629 } 2630 } 2631 2632 2633 static void oce_async_grp5_osbmc_process(POCE_SOFTC sc, 2634 struct oce_async_evt_grp5_os2bmc *evt) 2635 { 2636 DW_SWAP(evt, sizeof(struct oce_async_evt_grp5_os2bmc)); 2637 if (evt->u.s.mgmt_enable) 2638 sc->flags |= OCE_FLAGS_OS2BMC; 2639 else 2640 return; 2641 2642 sc->bmc_filt_mask = evt->u.s.arp_filter; 2643 sc->bmc_filt_mask |= (evt->u.s.dhcp_client_filt << 1); 2644 sc->bmc_filt_mask |= (evt->u.s.dhcp_server_filt << 2); 2645 sc->bmc_filt_mask |= (evt->u.s.net_bios_filt << 3); 2646 sc->bmc_filt_mask |= (evt->u.s.bcast_filt << 4); 2647 sc->bmc_filt_mask |= (evt->u.s.ipv6_nbr_filt << 5); 2648 sc->bmc_filt_mask |= (evt->u.s.ipv6_ra_filt << 6); 2649 sc->bmc_filt_mask |= (evt->u.s.ipv6_ras_filt << 7); 2650 sc->bmc_filt_mask |= (evt->u.s.mcast_filt << 8); 2651 } 2652 2653 2654 static void oce_process_grp5_events(POCE_SOFTC sc, struct oce_mq_cqe *cqe) 2655 { 2656 struct oce_async_event_grp5_pvid_state *gcqe; 2657 struct oce_async_evt_grp5_os2bmc *bmccqe; 2658 2659 switch (cqe->u0.s.async_type) { 2660 case ASYNC_EVENT_PVID_STATE: 2661 /* GRP5 PVID */ 2662 gcqe = (struct oce_async_event_grp5_pvid_state *)cqe; 2663 if (gcqe->enabled) 2664 sc->pvid = gcqe->tag & VLAN_VID_MASK; 2665 else 2666 sc->pvid = 0; 2667 break; 2668 case ASYNC_EVENT_OS2BMC: 2669 bmccqe = (struct oce_async_evt_grp5_os2bmc *)cqe; 2670 oce_async_grp5_osbmc_process(sc, bmccqe); 2671 break; 2672 default: 2673 break; 2674 } 2675 } 2676 2677 /* Handle the Completion Queue for the Mailbox/Async notifications */ 2678 uint16_t 2679 oce_mq_handler(void *arg) 2680 { 2681 struct oce_mq *mq = (struct oce_mq *)arg; 2682 POCE_SOFTC sc = mq->parent; 2683 struct oce_cq *cq = mq->cq; 2684 int num_cqes = 0, evt_type = 0, optype = 0; 2685 struct oce_mq_cqe *cqe; 2686 struct oce_async_cqe_link_state *acqe; 2687 struct oce_async_event_qnq *dbgcqe; 2688 2689 2690 bus_dmamap_sync(cq->ring->dma.tag, 2691 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2692 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe); 2693 2694 while (cqe->u0.dw[3]) { 2695 DW_SWAP((uint32_t *) cqe, sizeof(oce_mq_cqe)); 2696 if (cqe->u0.s.async_event) { 2697 evt_type = cqe->u0.s.event_type; 2698 optype = cqe->u0.s.async_type; 2699 if (evt_type == ASYNC_EVENT_CODE_LINK_STATE) { 2700 /* Link status evt */ 2701 acqe = (struct oce_async_cqe_link_state *)cqe; 2702 process_link_state(sc, acqe); 2703 } else if (evt_type == ASYNC_EVENT_GRP5) { 2704 oce_process_grp5_events(sc, cqe); 2705 } else if (evt_type == ASYNC_EVENT_CODE_DEBUG && 2706 optype == ASYNC_EVENT_DEBUG_QNQ) { 2707 dbgcqe = (struct oce_async_event_qnq *)cqe; 2708 if(dbgcqe->valid) 2709 sc->qnqid = dbgcqe->vlan_tag; 2710 sc->qnq_debug_event = TRUE; 2711 } 2712 } 2713 cqe->u0.dw[3] = 0; 2714 RING_GET(cq->ring, 1); 2715 bus_dmamap_sync(cq->ring->dma.tag, 2716 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2717 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe); 2718 num_cqes++; 2719 } 2720 2721 if (num_cqes) 2722 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); 2723 2724 return 0; 2725 } 2726 2727 2728 static void 2729 setup_max_queues_want(POCE_SOFTC sc) 2730 { 2731 /* Check if it is FLEX machine. Is so dont use RSS */ 2732 if ((sc->function_mode & FNM_FLEX10_MODE) || 2733 (sc->function_mode & FNM_UMC_MODE) || 2734 (sc->function_mode & FNM_VNIC_MODE) || 2735 (!is_rss_enabled(sc)) || 2736 IS_BE2(sc)) { 2737 sc->nrqs = 1; 2738 sc->nwqs = 1; 2739 } else { 2740 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1; 2741 sc->nwqs = MIN(OCE_NCPUS, sc->nrssqs); 2742 } 2743 2744 if (IS_BE2(sc) && is_rss_enabled(sc)) 2745 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1; 2746 } 2747 2748 2749 static void 2750 update_queues_got(POCE_SOFTC sc) 2751 { 2752 if (is_rss_enabled(sc)) { 2753 sc->nrqs = sc->intr_count + 1; 2754 sc->nwqs = sc->intr_count; 2755 } else { 2756 sc->nrqs = 1; 2757 sc->nwqs = 1; 2758 } 2759 2760 if (IS_BE2(sc)) 2761 sc->nwqs = 1; 2762 } 2763 2764 static int 2765 oce_check_ipv6_ext_hdr(struct mbuf *m) 2766 { 2767 struct ether_header *eh = mtod(m, struct ether_header *); 2768 caddr_t m_datatemp = m->m_data; 2769 2770 if (eh->ether_type == htons(ETHERTYPE_IPV6)) { 2771 m->m_data += sizeof(struct ether_header); 2772 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); 2773 2774 if((ip6->ip6_nxt != IPPROTO_TCP) && \ 2775 (ip6->ip6_nxt != IPPROTO_UDP)){ 2776 struct ip6_ext *ip6e = NULL; 2777 m->m_data += sizeof(struct ip6_hdr); 2778 2779 ip6e = (struct ip6_ext *) mtod(m, struct ip6_ext *); 2780 if(ip6e->ip6e_len == 0xff) { 2781 m->m_data = m_datatemp; 2782 return TRUE; 2783 } 2784 } 2785 m->m_data = m_datatemp; 2786 } 2787 return FALSE; 2788 } 2789 2790 static int 2791 is_be3_a1(POCE_SOFTC sc) 2792 { 2793 if((sc->flags & OCE_FLAGS_BE3) && ((sc->asic_revision & 0xFF) < 2)) { 2794 return TRUE; 2795 } 2796 return FALSE; 2797 } 2798 2799 static struct mbuf * 2800 oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete) 2801 { 2802 uint16_t vlan_tag = 0; 2803 2804 if(!M_WRITABLE(m)) 2805 return NULL; 2806 2807 /* Embed vlan tag in the packet if it is not part of it */ 2808 if(m->m_flags & M_VLANTAG) { 2809 vlan_tag = EVL_VLANOFTAG(m->m_pkthdr.ether_vtag); 2810 m->m_flags &= ~M_VLANTAG; 2811 } 2812 2813 /* if UMC, ignore vlan tag insertion and instead insert pvid */ 2814 if(sc->pvid) { 2815 if(!vlan_tag) 2816 vlan_tag = sc->pvid; 2817 if (complete) 2818 *complete = FALSE; 2819 } 2820 2821 if(vlan_tag) { 2822 m = ether_vlanencap(m, vlan_tag); 2823 } 2824 2825 if(sc->qnqid) { 2826 m = ether_vlanencap(m, sc->qnqid); 2827 2828 if (complete) 2829 *complete = FALSE; 2830 } 2831 return m; 2832 } 2833 2834 static int 2835 oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m) 2836 { 2837 if(is_be3_a1(sc) && IS_QNQ_OR_UMC(sc) && \ 2838 oce_check_ipv6_ext_hdr(m)) { 2839 return TRUE; 2840 } 2841 return FALSE; 2842 } 2843 2844 static void 2845 oce_get_config(POCE_SOFTC sc) 2846 { 2847 int rc = 0; 2848 uint32_t max_rss = 0; 2849 2850 if ((IS_BE(sc) || IS_SH(sc)) && (!sc->be3_native)) 2851 max_rss = OCE_LEGACY_MODE_RSS; 2852 else 2853 max_rss = OCE_MAX_RSS; 2854 2855 if (!IS_BE(sc)) { 2856 rc = oce_get_profile_config(sc, max_rss); 2857 if (rc) { 2858 sc->nwqs = OCE_MAX_WQ; 2859 sc->nrssqs = max_rss; 2860 sc->nrqs = sc->nrssqs + 1; 2861 } 2862 } 2863 else { /* For BE3 don't rely on fw for determining the resources */ 2864 sc->nrssqs = max_rss; 2865 sc->nrqs = sc->nrssqs + 1; 2866 sc->nwqs = OCE_MAX_WQ; 2867 sc->max_vlans = MAX_VLANFILTER_SIZE; 2868 } 2869 } 2870 2871 static void 2872 oce_rdma_close(void) 2873 { 2874 if (oce_rdma_if != NULL) { 2875 oce_rdma_if = NULL; 2876 } 2877 } 2878 2879 static void 2880 oce_get_mac_addr(POCE_SOFTC sc, uint8_t *macaddr) 2881 { 2882 memcpy(macaddr, sc->macaddr.mac_addr, 6); 2883 } 2884 2885 int 2886 oce_register_rdma(POCE_RDMA_INFO rdma_info, POCE_RDMA_IF rdma_if) 2887 { 2888 POCE_SOFTC sc; 2889 struct oce_dev_info di; 2890 int i; 2891 2892 if ((rdma_info == NULL) || (rdma_if == NULL)) { 2893 return -EINVAL; 2894 } 2895 2896 if ((rdma_info->size != OCE_RDMA_INFO_SIZE) || 2897 (rdma_if->size != OCE_RDMA_IF_SIZE)) { 2898 return -ENXIO; 2899 } 2900 2901 rdma_info->close = oce_rdma_close; 2902 rdma_info->mbox_post = oce_mbox_post; 2903 rdma_info->common_req_hdr_init = mbx_common_req_hdr_init; 2904 rdma_info->get_mac_addr = oce_get_mac_addr; 2905 2906 oce_rdma_if = rdma_if; 2907 2908 sc = softc_head; 2909 while (sc != NULL) { 2910 if (oce_rdma_if->announce != NULL) { 2911 memset(&di, 0, sizeof(di)); 2912 di.dev = sc->dev; 2913 di.softc = sc; 2914 di.ifp = sc->ifp; 2915 di.db_bhandle = sc->db_bhandle; 2916 di.db_btag = sc->db_btag; 2917 di.db_page_size = 4096; 2918 if (sc->flags & OCE_FLAGS_USING_MSIX) { 2919 di.intr_mode = OCE_INTERRUPT_MODE_MSIX; 2920 } else if (sc->flags & OCE_FLAGS_USING_MSI) { 2921 di.intr_mode = OCE_INTERRUPT_MODE_MSI; 2922 } else { 2923 di.intr_mode = OCE_INTERRUPT_MODE_INTX; 2924 } 2925 di.dev_family = OCE_GEN2_FAMILY; // fixme: must detect skyhawk 2926 if (di.intr_mode != OCE_INTERRUPT_MODE_INTX) { 2927 di.msix.num_vectors = sc->intr_count + sc->roce_intr_count; 2928 di.msix.start_vector = sc->intr_count; 2929 for (i=0; i<di.msix.num_vectors; i++) { 2930 di.msix.vector_list[i] = sc->intrs[i].vector; 2931 } 2932 } else { 2933 } 2934 memcpy(di.mac_addr, sc->macaddr.mac_addr, 6); 2935 di.vendor_id = pci_get_vendor(sc->dev); 2936 di.dev_id = pci_get_device(sc->dev); 2937 2938 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 2939 di.flags |= OCE_RDMA_INFO_RDMA_SUPPORTED; 2940 } 2941 2942 rdma_if->announce(&di); 2943 sc = sc->next; 2944 } 2945 } 2946 2947 return 0; 2948 } 2949 2950 static void 2951 oce_read_env_variables( POCE_SOFTC sc ) 2952 { 2953 char *value = NULL; 2954 int rc = 0; 2955 2956 /* read if user wants to enable hwlro or swlro */ 2957 //value = getenv("oce_enable_hwlro"); 2958 if(value && IS_SH(sc)) { 2959 sc->enable_hwlro = strtol(value, NULL, 10); 2960 if(sc->enable_hwlro) { 2961 rc = oce_mbox_nic_query_lro_capabilities(sc, NULL, NULL); 2962 if(rc) { 2963 device_printf(sc->dev, "no hardware lro support\n"); 2964 device_printf(sc->dev, "software lro enabled\n"); 2965 sc->enable_hwlro = 0; 2966 }else { 2967 device_printf(sc->dev, "hardware lro enabled\n"); 2968 oce_max_rsp_handled = 32; 2969 } 2970 }else { 2971 device_printf(sc->dev, "software lro enabled\n"); 2972 } 2973 }else { 2974 sc->enable_hwlro = 0; 2975 } 2976 2977 /* read mbuf size */ 2978 //value = getenv("oce_rq_buf_size"); 2979 if(value && IS_SH(sc)) { 2980 oce_rq_buf_size = strtol(value, NULL, 10); 2981 switch(oce_rq_buf_size) { 2982 case 2048: 2983 case 4096: 2984 case 9216: 2985 case 16384: 2986 break; 2987 2988 default: 2989 device_printf(sc->dev, " Supported oce_rq_buf_size values are 2K, 4K, 9K, 16K \n"); 2990 oce_rq_buf_size = 2048; 2991 } 2992 } 2993 2994 return; 2995 } 2996