1 /*- 2 * Copyright (C) 2013 Emulex 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * 3. Neither the name of the Emulex Corporation nor the names of its 16 * contributors may be used to endorse or promote products derived from 17 * this software without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * Contact Information: 32 * freebsd-drivers@emulex.com 33 * 34 * Emulex 35 * 3333 Susan Street 36 * Costa Mesa, CA 92626 37 */ 38 39 /* $FreeBSD$ */ 40 41 #include "opt_inet6.h" 42 #include "opt_inet.h" 43 44 #include "oce_if.h" 45 #include "oce_user.h" 46 47 #define is_tso_pkt(m) (m->m_pkthdr.csum_flags & CSUM_TSO) 48 49 /* UE Status Low CSR */ 50 static char *ue_status_low_desc[] = { 51 "CEV", 52 "CTX", 53 "DBUF", 54 "ERX", 55 "Host", 56 "MPU", 57 "NDMA", 58 "PTC ", 59 "RDMA ", 60 "RXF ", 61 "RXIPS ", 62 "RXULP0 ", 63 "RXULP1 ", 64 "RXULP2 ", 65 "TIM ", 66 "TPOST ", 67 "TPRE ", 68 "TXIPS ", 69 "TXULP0 ", 70 "TXULP1 ", 71 "UC ", 72 "WDMA ", 73 "TXULP2 ", 74 "HOST1 ", 75 "P0_OB_LINK ", 76 "P1_OB_LINK ", 77 "HOST_GPIO ", 78 "MBOX ", 79 "AXGMAC0", 80 "AXGMAC1", 81 "JTAG", 82 "MPU_INTPEND" 83 }; 84 85 /* UE Status High CSR */ 86 static char *ue_status_hi_desc[] = { 87 "LPCMEMHOST", 88 "MGMT_MAC", 89 "PCS0ONLINE", 90 "MPU_IRAM", 91 "PCS1ONLINE", 92 "PCTL0", 93 "PCTL1", 94 "PMEM", 95 "RR", 96 "TXPB", 97 "RXPP", 98 "XAUI", 99 "TXP", 100 "ARM", 101 "IPC", 102 "HOST2", 103 "HOST3", 104 "HOST4", 105 "HOST5", 106 "HOST6", 107 "HOST7", 108 "HOST8", 109 "HOST9", 110 "NETC", 111 "Unknown", 112 "Unknown", 113 "Unknown", 114 "Unknown", 115 "Unknown", 116 "Unknown", 117 "Unknown", 118 "Unknown" 119 }; 120 121 struct oce_common_cqe_info{ 122 uint8_t vtp:1; 123 uint8_t l4_cksum_pass:1; 124 uint8_t ip_cksum_pass:1; 125 uint8_t ipv6_frame:1; 126 uint8_t qnq:1; 127 uint8_t rsvd:3; 128 uint8_t num_frags; 129 uint16_t pkt_size; 130 uint16_t vtag; 131 }; 132 133 134 /* Driver entry points prototypes */ 135 static int oce_probe(device_t dev); 136 static int oce_attach(device_t dev); 137 static int oce_detach(device_t dev); 138 static int oce_shutdown(device_t dev); 139 static int oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data); 140 static void oce_init(void *xsc); 141 static int oce_multiq_start(struct ifnet *ifp, struct mbuf *m); 142 static void oce_multiq_flush(struct ifnet *ifp); 143 144 /* Driver interrupt routines protypes */ 145 static void oce_intr(void *arg, int pending); 146 static int oce_setup_intr(POCE_SOFTC sc); 147 static int oce_fast_isr(void *arg); 148 static int oce_alloc_intr(POCE_SOFTC sc, int vector, 149 void (*isr) (void *arg, int pending)); 150 151 /* Media callbacks prototypes */ 152 static void oce_media_status(struct ifnet *ifp, struct ifmediareq *req); 153 static int oce_media_change(struct ifnet *ifp); 154 155 /* Transmit routines prototypes */ 156 static int oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index); 157 static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq); 158 static void oce_process_tx_completion(struct oce_wq *wq); 159 static int oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, 160 struct oce_wq *wq); 161 162 /* Receive routines prototypes */ 163 static int oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe); 164 static int oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe); 165 static void oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe); 166 static void oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq); 167 static uint16_t oce_rq_handler_lro(void *arg); 168 static void oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2); 169 static void oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2); 170 static void oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m); 171 172 /* Helper function prototypes in this file */ 173 static int oce_attach_ifp(POCE_SOFTC sc); 174 static void oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag); 175 static void oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag); 176 static int oce_vid_config(POCE_SOFTC sc); 177 static void oce_mac_addr_set(POCE_SOFTC sc); 178 static int oce_handle_passthrough(struct ifnet *ifp, caddr_t data); 179 static void oce_local_timer(void *arg); 180 static void oce_if_deactivate(POCE_SOFTC sc); 181 static void oce_if_activate(POCE_SOFTC sc); 182 static void setup_max_queues_want(POCE_SOFTC sc); 183 static void update_queues_got(POCE_SOFTC sc); 184 static void process_link_state(POCE_SOFTC sc, 185 struct oce_async_cqe_link_state *acqe); 186 static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m); 187 static void oce_get_config(POCE_SOFTC sc); 188 static struct mbuf *oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete); 189 static void oce_read_env_variables(POCE_SOFTC sc); 190 191 192 /* IP specific */ 193 #if defined(INET6) || defined(INET) 194 static int oce_init_lro(POCE_SOFTC sc); 195 static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp); 196 #endif 197 198 static device_method_t oce_dispatch[] = { 199 DEVMETHOD(device_probe, oce_probe), 200 DEVMETHOD(device_attach, oce_attach), 201 DEVMETHOD(device_detach, oce_detach), 202 DEVMETHOD(device_shutdown, oce_shutdown), 203 204 DEVMETHOD_END 205 }; 206 207 static driver_t oce_driver = { 208 "oce", 209 oce_dispatch, 210 sizeof(OCE_SOFTC) 211 }; 212 static devclass_t oce_devclass; 213 214 215 DRIVER_MODULE(oce, pci, oce_driver, oce_devclass, 0, 0); 216 MODULE_DEPEND(oce, pci, 1, 1, 1); 217 MODULE_DEPEND(oce, ether, 1, 1, 1); 218 MODULE_VERSION(oce, 1); 219 220 221 /* global vars */ 222 const char component_revision[32] = {"///" COMPONENT_REVISION "///"}; 223 224 /* Module capabilites and parameters */ 225 uint32_t oce_max_rsp_handled = OCE_MAX_RSP_HANDLED; 226 uint32_t oce_enable_rss = OCE_MODCAP_RSS; 227 uint32_t oce_rq_buf_size = 2048; 228 229 TUNABLE_INT("hw.oce.max_rsp_handled", &oce_max_rsp_handled); 230 TUNABLE_INT("hw.oce.enable_rss", &oce_enable_rss); 231 232 233 /* Supported devices table */ 234 static uint32_t supportedDevices[] = { 235 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE2, 236 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE3, 237 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_BE3, 238 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201, 239 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201_VF, 240 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_SH 241 }; 242 243 POCE_SOFTC softc_head = NULL; 244 POCE_SOFTC softc_tail = NULL; 245 246 struct oce_rdma_if *oce_rdma_if = NULL; 247 248 /***************************************************************************** 249 * Driver entry points functions * 250 *****************************************************************************/ 251 252 static int 253 oce_probe(device_t dev) 254 { 255 uint16_t vendor = 0; 256 uint16_t device = 0; 257 int i = 0; 258 char str[256] = {0}; 259 POCE_SOFTC sc; 260 261 sc = device_get_softc(dev); 262 bzero(sc, sizeof(OCE_SOFTC)); 263 sc->dev = dev; 264 265 vendor = pci_get_vendor(dev); 266 device = pci_get_device(dev); 267 268 for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint32_t)); i++) { 269 if (vendor == ((supportedDevices[i] >> 16) & 0xffff)) { 270 if (device == (supportedDevices[i] & 0xffff)) { 271 sprintf(str, "%s:%s", "Emulex CNA NIC function", 272 component_revision); 273 device_set_desc_copy(dev, str); 274 275 switch (device) { 276 case PCI_PRODUCT_BE2: 277 sc->flags |= OCE_FLAGS_BE2; 278 break; 279 case PCI_PRODUCT_BE3: 280 sc->flags |= OCE_FLAGS_BE3; 281 break; 282 case PCI_PRODUCT_XE201: 283 case PCI_PRODUCT_XE201_VF: 284 sc->flags |= OCE_FLAGS_XE201; 285 break; 286 case PCI_PRODUCT_SH: 287 sc->flags |= OCE_FLAGS_SH; 288 break; 289 default: 290 return ENXIO; 291 } 292 return BUS_PROBE_DEFAULT; 293 } 294 } 295 } 296 297 return ENXIO; 298 } 299 300 301 static int 302 oce_attach(device_t dev) 303 { 304 POCE_SOFTC sc; 305 int rc = 0; 306 307 sc = device_get_softc(dev); 308 309 rc = oce_hw_pci_alloc(sc); 310 if (rc) 311 return rc; 312 313 sc->tx_ring_size = OCE_TX_RING_SIZE; 314 sc->rx_ring_size = OCE_RX_RING_SIZE; 315 /* receive fragment size should be multiple of 2K */ 316 sc->rq_frag_size = ((oce_rq_buf_size / 2048) * 2048); 317 sc->flow_control = OCE_DEFAULT_FLOW_CONTROL; 318 sc->promisc = OCE_DEFAULT_PROMISCUOUS; 319 320 LOCK_CREATE(&sc->bmbx_lock, "Mailbox_lock"); 321 LOCK_CREATE(&sc->dev_lock, "Device_lock"); 322 323 /* initialise the hardware */ 324 rc = oce_hw_init(sc); 325 if (rc) 326 goto pci_res_free; 327 328 oce_read_env_variables(sc); 329 330 oce_get_config(sc); 331 332 setup_max_queues_want(sc); 333 334 rc = oce_setup_intr(sc); 335 if (rc) 336 goto mbox_free; 337 338 rc = oce_queue_init_all(sc); 339 if (rc) 340 goto intr_free; 341 342 rc = oce_attach_ifp(sc); 343 if (rc) 344 goto queues_free; 345 346 #if defined(INET6) || defined(INET) 347 rc = oce_init_lro(sc); 348 if (rc) 349 goto ifp_free; 350 #endif 351 352 rc = oce_hw_start(sc); 353 if (rc) 354 goto lro_free; 355 356 sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 357 oce_add_vlan, sc, EVENTHANDLER_PRI_FIRST); 358 sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 359 oce_del_vlan, sc, EVENTHANDLER_PRI_FIRST); 360 361 rc = oce_stats_init(sc); 362 if (rc) 363 goto vlan_free; 364 365 oce_add_sysctls(sc); 366 367 callout_init(&sc->timer, CALLOUT_MPSAFE); 368 rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc); 369 if (rc) 370 goto stats_free; 371 372 sc->next =NULL; 373 if (softc_tail != NULL) { 374 softc_tail->next = sc; 375 } else { 376 softc_head = sc; 377 } 378 softc_tail = sc; 379 380 return 0; 381 382 stats_free: 383 callout_drain(&sc->timer); 384 oce_stats_free(sc); 385 vlan_free: 386 if (sc->vlan_attach) 387 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); 388 if (sc->vlan_detach) 389 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); 390 oce_hw_intr_disable(sc); 391 lro_free: 392 #if defined(INET6) || defined(INET) 393 oce_free_lro(sc); 394 ifp_free: 395 #endif 396 ether_ifdetach(sc->ifp); 397 if_free(sc->ifp); 398 queues_free: 399 oce_queue_release_all(sc); 400 intr_free: 401 oce_intr_free(sc); 402 mbox_free: 403 oce_dma_free(sc, &sc->bsmbx); 404 pci_res_free: 405 oce_hw_pci_free(sc); 406 LOCK_DESTROY(&sc->dev_lock); 407 LOCK_DESTROY(&sc->bmbx_lock); 408 return rc; 409 410 } 411 412 413 static int 414 oce_detach(device_t dev) 415 { 416 POCE_SOFTC sc = device_get_softc(dev); 417 POCE_SOFTC poce_sc_tmp, *ppoce_sc_tmp1, poce_sc_tmp2 = NULL; 418 419 poce_sc_tmp = softc_head; 420 ppoce_sc_tmp1 = &softc_head; 421 while (poce_sc_tmp != NULL) { 422 if (poce_sc_tmp == sc) { 423 *ppoce_sc_tmp1 = sc->next; 424 if (sc->next == NULL) { 425 softc_tail = poce_sc_tmp2; 426 } 427 break; 428 } 429 poce_sc_tmp2 = poce_sc_tmp; 430 ppoce_sc_tmp1 = &poce_sc_tmp->next; 431 poce_sc_tmp = poce_sc_tmp->next; 432 } 433 434 LOCK(&sc->dev_lock); 435 oce_if_deactivate(sc); 436 UNLOCK(&sc->dev_lock); 437 438 callout_drain(&sc->timer); 439 440 if (sc->vlan_attach != NULL) 441 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); 442 if (sc->vlan_detach != NULL) 443 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); 444 445 ether_ifdetach(sc->ifp); 446 447 if_free(sc->ifp); 448 449 oce_hw_shutdown(sc); 450 451 bus_generic_detach(dev); 452 453 return 0; 454 } 455 456 457 static int 458 oce_shutdown(device_t dev) 459 { 460 int rc; 461 462 rc = oce_detach(dev); 463 464 return rc; 465 } 466 467 468 static int 469 oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 470 { 471 struct ifreq *ifr = (struct ifreq *)data; 472 POCE_SOFTC sc = ifp->if_softc; 473 int rc = 0; 474 uint32_t u; 475 476 switch (command) { 477 478 case SIOCGIFMEDIA: 479 rc = ifmedia_ioctl(ifp, ifr, &sc->media, command); 480 break; 481 482 case SIOCSIFMTU: 483 if (ifr->ifr_mtu > OCE_MAX_MTU) 484 rc = EINVAL; 485 else 486 ifp->if_mtu = ifr->ifr_mtu; 487 break; 488 489 case SIOCSIFFLAGS: 490 if (ifp->if_flags & IFF_UP) { 491 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 492 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; 493 oce_init(sc); 494 } 495 device_printf(sc->dev, "Interface Up\n"); 496 } else { 497 LOCK(&sc->dev_lock); 498 499 sc->ifp->if_drv_flags &= 500 ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 501 oce_if_deactivate(sc); 502 503 UNLOCK(&sc->dev_lock); 504 505 device_printf(sc->dev, "Interface Down\n"); 506 } 507 508 if ((ifp->if_flags & IFF_PROMISC) && !sc->promisc) { 509 if (!oce_rxf_set_promiscuous(sc, (1 | (1 << 1)))) 510 sc->promisc = TRUE; 511 } else if (!(ifp->if_flags & IFF_PROMISC) && sc->promisc) { 512 if (!oce_rxf_set_promiscuous(sc, 0)) 513 sc->promisc = FALSE; 514 } 515 516 break; 517 518 case SIOCADDMULTI: 519 case SIOCDELMULTI: 520 rc = oce_hw_update_multicast(sc); 521 if (rc) 522 device_printf(sc->dev, 523 "Update multicast address failed\n"); 524 break; 525 526 case SIOCSIFCAP: 527 u = ifr->ifr_reqcap ^ ifp->if_capenable; 528 529 if (u & IFCAP_TXCSUM) { 530 ifp->if_capenable ^= IFCAP_TXCSUM; 531 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 532 533 if (IFCAP_TSO & ifp->if_capenable && 534 !(IFCAP_TXCSUM & ifp->if_capenable)) { 535 ifp->if_capenable &= ~IFCAP_TSO; 536 ifp->if_hwassist &= ~CSUM_TSO; 537 if_printf(ifp, 538 "TSO disabled due to -txcsum.\n"); 539 } 540 } 541 542 if (u & IFCAP_RXCSUM) 543 ifp->if_capenable ^= IFCAP_RXCSUM; 544 545 if (u & IFCAP_TSO4) { 546 ifp->if_capenable ^= IFCAP_TSO4; 547 548 if (IFCAP_TSO & ifp->if_capenable) { 549 if (IFCAP_TXCSUM & ifp->if_capenable) 550 ifp->if_hwassist |= CSUM_TSO; 551 else { 552 ifp->if_capenable &= ~IFCAP_TSO; 553 ifp->if_hwassist &= ~CSUM_TSO; 554 if_printf(ifp, 555 "Enable txcsum first.\n"); 556 rc = EAGAIN; 557 } 558 } else 559 ifp->if_hwassist &= ~CSUM_TSO; 560 } 561 562 if (u & IFCAP_VLAN_HWTAGGING) 563 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 564 565 if (u & IFCAP_VLAN_HWFILTER) { 566 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 567 oce_vid_config(sc); 568 } 569 #if defined(INET6) || defined(INET) 570 if (u & IFCAP_LRO) { 571 ifp->if_capenable ^= IFCAP_LRO; 572 if(sc->enable_hwlro) { 573 if(ifp->if_capenable & IFCAP_LRO) { 574 rc = oce_mbox_nic_set_iface_lro_config(sc, 1); 575 }else { 576 rc = oce_mbox_nic_set_iface_lro_config(sc, 0); 577 } 578 } 579 } 580 #endif 581 582 break; 583 584 case SIOCGPRIVATE_0: 585 rc = oce_handle_passthrough(ifp, data); 586 break; 587 default: 588 rc = ether_ioctl(ifp, command, data); 589 break; 590 } 591 592 return rc; 593 } 594 595 596 static void 597 oce_init(void *arg) 598 { 599 POCE_SOFTC sc = arg; 600 601 LOCK(&sc->dev_lock); 602 603 if (sc->ifp->if_flags & IFF_UP) { 604 oce_if_deactivate(sc); 605 oce_if_activate(sc); 606 } 607 608 UNLOCK(&sc->dev_lock); 609 610 } 611 612 613 static int 614 oce_multiq_start(struct ifnet *ifp, struct mbuf *m) 615 { 616 POCE_SOFTC sc = ifp->if_softc; 617 struct oce_wq *wq = NULL; 618 int queue_index = 0; 619 int status = 0; 620 621 if (!sc->link_status) 622 return ENXIO; 623 624 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 625 queue_index = m->m_pkthdr.flowid % sc->nwqs; 626 627 wq = sc->wq[queue_index]; 628 629 LOCK(&wq->tx_lock); 630 status = oce_multiq_transmit(ifp, m, wq); 631 UNLOCK(&wq->tx_lock); 632 633 return status; 634 635 } 636 637 638 static void 639 oce_multiq_flush(struct ifnet *ifp) 640 { 641 POCE_SOFTC sc = ifp->if_softc; 642 struct mbuf *m; 643 int i = 0; 644 645 for (i = 0; i < sc->nwqs; i++) { 646 while ((m = buf_ring_dequeue_sc(sc->wq[i]->br)) != NULL) 647 m_freem(m); 648 } 649 if_qflush(ifp); 650 } 651 652 653 654 /***************************************************************************** 655 * Driver interrupt routines functions * 656 *****************************************************************************/ 657 658 static void 659 oce_intr(void *arg, int pending) 660 { 661 662 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg; 663 POCE_SOFTC sc = ii->sc; 664 struct oce_eq *eq = ii->eq; 665 struct oce_eqe *eqe; 666 struct oce_cq *cq = NULL; 667 int i, num_eqes = 0; 668 669 670 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, 671 BUS_DMASYNC_POSTWRITE); 672 do { 673 eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe); 674 if (eqe->evnt == 0) 675 break; 676 eqe->evnt = 0; 677 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, 678 BUS_DMASYNC_POSTWRITE); 679 RING_GET(eq->ring, 1); 680 num_eqes++; 681 682 } while (TRUE); 683 684 if (!num_eqes) 685 goto eq_arm; /* Spurious */ 686 687 /* Clear EQ entries, but dont arm */ 688 oce_arm_eq(sc, eq->eq_id, num_eqes, FALSE, FALSE); 689 690 /* Process TX, RX and MCC. But dont arm CQ*/ 691 for (i = 0; i < eq->cq_valid; i++) { 692 cq = eq->cq[i]; 693 (*cq->cq_handler)(cq->cb_arg); 694 } 695 696 /* Arm all cqs connected to this EQ */ 697 for (i = 0; i < eq->cq_valid; i++) { 698 cq = eq->cq[i]; 699 oce_arm_cq(sc, cq->cq_id, 0, TRUE); 700 } 701 702 eq_arm: 703 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); 704 705 return; 706 } 707 708 709 static int 710 oce_setup_intr(POCE_SOFTC sc) 711 { 712 int rc = 0, use_intx = 0; 713 int vector = 0, req_vectors = 0; 714 int tot_req_vectors, tot_vectors; 715 716 if (is_rss_enabled(sc)) 717 req_vectors = MAX((sc->nrqs - 1), sc->nwqs); 718 else 719 req_vectors = 1; 720 721 tot_req_vectors = req_vectors; 722 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 723 if (req_vectors > 1) { 724 tot_req_vectors += OCE_RDMA_VECTORS; 725 sc->roce_intr_count = OCE_RDMA_VECTORS; 726 } 727 } 728 729 if (sc->flags & OCE_FLAGS_MSIX_CAPABLE) { 730 sc->intr_count = req_vectors; 731 tot_vectors = tot_req_vectors; 732 rc = pci_alloc_msix(sc->dev, &tot_vectors); 733 if (rc != 0) { 734 use_intx = 1; 735 pci_release_msi(sc->dev); 736 } else { 737 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 738 if (tot_vectors < tot_req_vectors) { 739 if (sc->intr_count < (2 * OCE_RDMA_VECTORS)) { 740 sc->roce_intr_count = (tot_vectors / 2); 741 } 742 sc->intr_count = tot_vectors - sc->roce_intr_count; 743 } 744 } else { 745 sc->intr_count = tot_vectors; 746 } 747 sc->flags |= OCE_FLAGS_USING_MSIX; 748 } 749 } else 750 use_intx = 1; 751 752 if (use_intx) 753 sc->intr_count = 1; 754 755 /* Scale number of queues based on intr we got */ 756 update_queues_got(sc); 757 758 if (use_intx) { 759 device_printf(sc->dev, "Using legacy interrupt\n"); 760 rc = oce_alloc_intr(sc, vector, oce_intr); 761 if (rc) 762 goto error; 763 } else { 764 for (; vector < sc->intr_count; vector++) { 765 rc = oce_alloc_intr(sc, vector, oce_intr); 766 if (rc) 767 goto error; 768 } 769 } 770 771 return 0; 772 error: 773 oce_intr_free(sc); 774 return rc; 775 } 776 777 778 static int 779 oce_fast_isr(void *arg) 780 { 781 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg; 782 POCE_SOFTC sc = ii->sc; 783 784 if (ii->eq == NULL) 785 return FILTER_STRAY; 786 787 oce_arm_eq(sc, ii->eq->eq_id, 0, FALSE, TRUE); 788 789 taskqueue_enqueue(ii->tq, &ii->task); 790 791 ii->eq->intr++; 792 793 return FILTER_HANDLED; 794 } 795 796 797 static int 798 oce_alloc_intr(POCE_SOFTC sc, int vector, void (*isr) (void *arg, int pending)) 799 { 800 POCE_INTR_INFO ii = &sc->intrs[vector]; 801 int rc = 0, rr; 802 803 if (vector >= OCE_MAX_EQ) 804 return (EINVAL); 805 806 /* Set the resource id for the interrupt. 807 * MSIx is vector + 1 for the resource id, 808 * INTx is 0 for the resource id. 809 */ 810 if (sc->flags & OCE_FLAGS_USING_MSIX) 811 rr = vector + 1; 812 else 813 rr = 0; 814 ii->intr_res = bus_alloc_resource_any(sc->dev, 815 SYS_RES_IRQ, 816 &rr, RF_ACTIVE|RF_SHAREABLE); 817 ii->irq_rr = rr; 818 if (ii->intr_res == NULL) { 819 device_printf(sc->dev, 820 "Could not allocate interrupt\n"); 821 rc = ENXIO; 822 return rc; 823 } 824 825 TASK_INIT(&ii->task, 0, isr, ii); 826 ii->vector = vector; 827 sprintf(ii->task_name, "oce_task[%d]", ii->vector); 828 ii->tq = taskqueue_create_fast(ii->task_name, 829 M_NOWAIT, 830 taskqueue_thread_enqueue, 831 &ii->tq); 832 taskqueue_start_threads(&ii->tq, 1, PI_NET, "%s taskq", 833 device_get_nameunit(sc->dev)); 834 835 ii->sc = sc; 836 rc = bus_setup_intr(sc->dev, 837 ii->intr_res, 838 INTR_TYPE_NET, 839 oce_fast_isr, NULL, ii, &ii->tag); 840 return rc; 841 842 } 843 844 845 void 846 oce_intr_free(POCE_SOFTC sc) 847 { 848 int i = 0; 849 850 for (i = 0; i < sc->intr_count; i++) { 851 852 if (sc->intrs[i].tag != NULL) 853 bus_teardown_intr(sc->dev, sc->intrs[i].intr_res, 854 sc->intrs[i].tag); 855 if (sc->intrs[i].tq != NULL) 856 taskqueue_free(sc->intrs[i].tq); 857 858 if (sc->intrs[i].intr_res != NULL) 859 bus_release_resource(sc->dev, SYS_RES_IRQ, 860 sc->intrs[i].irq_rr, 861 sc->intrs[i].intr_res); 862 sc->intrs[i].tag = NULL; 863 sc->intrs[i].intr_res = NULL; 864 } 865 866 if (sc->flags & OCE_FLAGS_USING_MSIX) 867 pci_release_msi(sc->dev); 868 869 } 870 871 872 873 /****************************************************************************** 874 * Media callbacks functions * 875 ******************************************************************************/ 876 877 static void 878 oce_media_status(struct ifnet *ifp, struct ifmediareq *req) 879 { 880 POCE_SOFTC sc = (POCE_SOFTC) ifp->if_softc; 881 882 883 req->ifm_status = IFM_AVALID; 884 req->ifm_active = IFM_ETHER; 885 886 if (sc->link_status == 1) 887 req->ifm_status |= IFM_ACTIVE; 888 else 889 return; 890 891 switch (sc->link_speed) { 892 case 1: /* 10 Mbps */ 893 req->ifm_active |= IFM_10_T | IFM_FDX; 894 sc->speed = 10; 895 break; 896 case 2: /* 100 Mbps */ 897 req->ifm_active |= IFM_100_TX | IFM_FDX; 898 sc->speed = 100; 899 break; 900 case 3: /* 1 Gbps */ 901 req->ifm_active |= IFM_1000_T | IFM_FDX; 902 sc->speed = 1000; 903 break; 904 case 4: /* 10 Gbps */ 905 req->ifm_active |= IFM_10G_SR | IFM_FDX; 906 sc->speed = 10000; 907 break; 908 case 5: /* 20 Gbps */ 909 req->ifm_active |= IFM_10G_SR | IFM_FDX; 910 sc->speed = 20000; 911 break; 912 case 6: /* 25 Gbps */ 913 req->ifm_active |= IFM_10G_SR | IFM_FDX; 914 sc->speed = 25000; 915 break; 916 case 7: /* 40 Gbps */ 917 req->ifm_active |= IFM_40G_SR4 | IFM_FDX; 918 sc->speed = 40000; 919 break; 920 default: 921 sc->speed = 0; 922 break; 923 } 924 925 return; 926 } 927 928 929 int 930 oce_media_change(struct ifnet *ifp) 931 { 932 return 0; 933 } 934 935 936 static void oce_is_pkt_dest_bmc(POCE_SOFTC sc, 937 struct mbuf *m, boolean_t *os2bmc, 938 struct mbuf **m_new) 939 { 940 struct ether_header *eh = NULL; 941 942 eh = mtod(m, struct ether_header *); 943 944 if (!is_os2bmc_enabled(sc) || *os2bmc) { 945 *os2bmc = FALSE; 946 goto done; 947 } 948 if (!ETHER_IS_MULTICAST(eh->ether_dhost)) 949 goto done; 950 951 if (is_mc_allowed_on_bmc(sc, eh) || 952 is_bc_allowed_on_bmc(sc, eh) || 953 is_arp_allowed_on_bmc(sc, ntohs(eh->ether_type))) { 954 *os2bmc = TRUE; 955 goto done; 956 } 957 958 if (mtod(m, struct ip *)->ip_p == IPPROTO_IPV6) { 959 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); 960 uint8_t nexthdr = ip6->ip6_nxt; 961 if (nexthdr == IPPROTO_ICMPV6) { 962 struct icmp6_hdr *icmp6 = (struct icmp6_hdr *)(ip6 + 1); 963 switch (icmp6->icmp6_type) { 964 case ND_ROUTER_ADVERT: 965 *os2bmc = is_ipv6_ra_filt_enabled(sc); 966 goto done; 967 case ND_NEIGHBOR_ADVERT: 968 *os2bmc = is_ipv6_na_filt_enabled(sc); 969 goto done; 970 default: 971 break; 972 } 973 } 974 } 975 976 if (mtod(m, struct ip *)->ip_p == IPPROTO_UDP) { 977 struct ip *ip = mtod(m, struct ip *); 978 int iphlen = ip->ip_hl << 2; 979 struct udphdr *uh = (struct udphdr *)((caddr_t)ip + iphlen); 980 switch (uh->uh_dport) { 981 case DHCP_CLIENT_PORT: 982 *os2bmc = is_dhcp_client_filt_enabled(sc); 983 goto done; 984 case DHCP_SERVER_PORT: 985 *os2bmc = is_dhcp_srvr_filt_enabled(sc); 986 goto done; 987 case NET_BIOS_PORT1: 988 case NET_BIOS_PORT2: 989 *os2bmc = is_nbios_filt_enabled(sc); 990 goto done; 991 case DHCPV6_RAS_PORT: 992 *os2bmc = is_ipv6_ras_filt_enabled(sc); 993 goto done; 994 default: 995 break; 996 } 997 } 998 done: 999 if (*os2bmc) { 1000 *m_new = m_dup(m, M_NOWAIT); 1001 if (!*m_new) { 1002 *os2bmc = FALSE; 1003 return; 1004 } 1005 *m_new = oce_insert_vlan_tag(sc, *m_new, NULL); 1006 } 1007 } 1008 1009 1010 1011 /***************************************************************************** 1012 * Transmit routines functions * 1013 *****************************************************************************/ 1014 1015 static int 1016 oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index) 1017 { 1018 int rc = 0, i, retry_cnt = 0; 1019 bus_dma_segment_t segs[OCE_MAX_TX_ELEMENTS]; 1020 struct mbuf *m, *m_temp, *m_new = NULL; 1021 struct oce_wq *wq = sc->wq[wq_index]; 1022 struct oce_packet_desc *pd; 1023 struct oce_nic_hdr_wqe *nichdr; 1024 struct oce_nic_frag_wqe *nicfrag; 1025 struct ether_header *eh = NULL; 1026 int num_wqes; 1027 uint32_t reg_value; 1028 boolean_t complete = TRUE; 1029 boolean_t os2bmc = FALSE; 1030 1031 m = *mpp; 1032 if (!m) 1033 return EINVAL; 1034 1035 if (!(m->m_flags & M_PKTHDR)) { 1036 rc = ENXIO; 1037 goto free_ret; 1038 } 1039 1040 /* Don't allow non-TSO packets longer than MTU */ 1041 if (!is_tso_pkt(m)) { 1042 eh = mtod(m, struct ether_header *); 1043 if(m->m_pkthdr.len > ETHER_MAX_FRAME(sc->ifp, eh->ether_type, FALSE)) 1044 goto free_ret; 1045 } 1046 1047 if(oce_tx_asic_stall_verify(sc, m)) { 1048 m = oce_insert_vlan_tag(sc, m, &complete); 1049 if(!m) { 1050 device_printf(sc->dev, "Insertion unsuccessful\n"); 1051 return 0; 1052 } 1053 1054 } 1055 1056 /* Lancer, SH ASIC has a bug wherein Packets that are 32 bytes or less 1057 * may cause a transmit stall on that port. So the work-around is to 1058 * pad short packets (<= 32 bytes) to a 36-byte length. 1059 */ 1060 if(IS_SH(sc) || IS_XE201(sc) ) { 1061 if(m->m_pkthdr.len <= 32) { 1062 char buf[36]; 1063 bzero((void *)buf, 36); 1064 m_append(m, (36 - m->m_pkthdr.len), buf); 1065 } 1066 } 1067 1068 tx_start: 1069 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1070 /* consolidate packet buffers for TSO/LSO segment offload */ 1071 #if defined(INET6) || defined(INET) 1072 m = oce_tso_setup(sc, mpp); 1073 #else 1074 m = NULL; 1075 #endif 1076 if (m == NULL) { 1077 rc = ENXIO; 1078 goto free_ret; 1079 } 1080 } 1081 1082 1083 pd = &wq->pckts[wq->pkt_desc_head]; 1084 1085 retry: 1086 rc = bus_dmamap_load_mbuf_sg(wq->tag, 1087 pd->map, 1088 m, segs, &pd->nsegs, BUS_DMA_NOWAIT); 1089 if (rc == 0) { 1090 num_wqes = pd->nsegs + 1; 1091 if (IS_BE(sc) || IS_SH(sc)) { 1092 /*Dummy required only for BE3.*/ 1093 if (num_wqes & 1) 1094 num_wqes++; 1095 } 1096 if (num_wqes >= RING_NUM_FREE(wq->ring)) { 1097 bus_dmamap_unload(wq->tag, pd->map); 1098 return EBUSY; 1099 } 1100 atomic_store_rel_int(&wq->pkt_desc_head, 1101 (wq->pkt_desc_head + 1) % \ 1102 OCE_WQ_PACKET_ARRAY_SIZE); 1103 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_PREWRITE); 1104 pd->mbuf = m; 1105 1106 nichdr = 1107 RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_hdr_wqe); 1108 nichdr->u0.dw[0] = 0; 1109 nichdr->u0.dw[1] = 0; 1110 nichdr->u0.dw[2] = 0; 1111 nichdr->u0.dw[3] = 0; 1112 1113 nichdr->u0.s.complete = complete; 1114 nichdr->u0.s.mgmt = os2bmc; 1115 nichdr->u0.s.event = 1; 1116 nichdr->u0.s.crc = 1; 1117 nichdr->u0.s.forward = 0; 1118 nichdr->u0.s.ipcs = (m->m_pkthdr.csum_flags & CSUM_IP) ? 1 : 0; 1119 nichdr->u0.s.udpcs = 1120 (m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0; 1121 nichdr->u0.s.tcpcs = 1122 (m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0; 1123 nichdr->u0.s.num_wqe = num_wqes; 1124 nichdr->u0.s.total_length = m->m_pkthdr.len; 1125 1126 if (m->m_flags & M_VLANTAG) { 1127 nichdr->u0.s.vlan = 1; /*Vlan present*/ 1128 nichdr->u0.s.vlan_tag = m->m_pkthdr.ether_vtag; 1129 } 1130 1131 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1132 if (m->m_pkthdr.tso_segsz) { 1133 nichdr->u0.s.lso = 1; 1134 nichdr->u0.s.lso_mss = m->m_pkthdr.tso_segsz; 1135 } 1136 if (!IS_BE(sc) || !IS_SH(sc)) 1137 nichdr->u0.s.ipcs = 1; 1138 } 1139 1140 RING_PUT(wq->ring, 1); 1141 atomic_add_int(&wq->ring->num_used, 1); 1142 1143 for (i = 0; i < pd->nsegs; i++) { 1144 nicfrag = 1145 RING_GET_PRODUCER_ITEM_VA(wq->ring, 1146 struct oce_nic_frag_wqe); 1147 nicfrag->u0.s.rsvd0 = 0; 1148 nicfrag->u0.s.frag_pa_hi = ADDR_HI(segs[i].ds_addr); 1149 nicfrag->u0.s.frag_pa_lo = ADDR_LO(segs[i].ds_addr); 1150 nicfrag->u0.s.frag_len = segs[i].ds_len; 1151 pd->wqe_idx = wq->ring->pidx; 1152 RING_PUT(wq->ring, 1); 1153 atomic_add_int(&wq->ring->num_used, 1); 1154 } 1155 if (num_wqes > (pd->nsegs + 1)) { 1156 nicfrag = 1157 RING_GET_PRODUCER_ITEM_VA(wq->ring, 1158 struct oce_nic_frag_wqe); 1159 nicfrag->u0.dw[0] = 0; 1160 nicfrag->u0.dw[1] = 0; 1161 nicfrag->u0.dw[2] = 0; 1162 nicfrag->u0.dw[3] = 0; 1163 pd->wqe_idx = wq->ring->pidx; 1164 RING_PUT(wq->ring, 1); 1165 atomic_add_int(&wq->ring->num_used, 1); 1166 pd->nsegs++; 1167 } 1168 1169 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1); 1170 wq->tx_stats.tx_reqs++; 1171 wq->tx_stats.tx_wrbs += num_wqes; 1172 wq->tx_stats.tx_bytes += m->m_pkthdr.len; 1173 wq->tx_stats.tx_pkts++; 1174 1175 bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map, 1176 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1177 reg_value = (num_wqes << 16) | wq->wq_id; 1178 1179 /* if os2bmc is not enabled or if the pkt is already tagged as 1180 bmc, do nothing 1181 */ 1182 oce_is_pkt_dest_bmc(sc, m, &os2bmc, &m_new); 1183 1184 OCE_WRITE_REG32(sc, db, wq->db_offset, reg_value); 1185 1186 } else if (rc == EFBIG) { 1187 if (retry_cnt == 0) { 1188 m_temp = m_defrag(m, M_NOWAIT); 1189 if (m_temp == NULL) 1190 goto free_ret; 1191 m = m_temp; 1192 *mpp = m_temp; 1193 retry_cnt = retry_cnt + 1; 1194 goto retry; 1195 } else 1196 goto free_ret; 1197 } else if (rc == ENOMEM) 1198 return rc; 1199 else 1200 goto free_ret; 1201 1202 if (os2bmc) { 1203 m = m_new; 1204 goto tx_start; 1205 } 1206 1207 return 0; 1208 1209 free_ret: 1210 m_freem(*mpp); 1211 *mpp = NULL; 1212 return rc; 1213 } 1214 1215 1216 static void 1217 oce_process_tx_completion(struct oce_wq *wq) 1218 { 1219 struct oce_packet_desc *pd; 1220 POCE_SOFTC sc = (POCE_SOFTC) wq->parent; 1221 struct mbuf *m; 1222 1223 pd = &wq->pckts[wq->pkt_desc_tail]; 1224 atomic_store_rel_int(&wq->pkt_desc_tail, 1225 (wq->pkt_desc_tail + 1) % OCE_WQ_PACKET_ARRAY_SIZE); 1226 atomic_subtract_int(&wq->ring->num_used, pd->nsegs + 1); 1227 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1228 bus_dmamap_unload(wq->tag, pd->map); 1229 1230 m = pd->mbuf; 1231 m_freem(m); 1232 pd->mbuf = NULL; 1233 1234 1235 if (sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) { 1236 if (wq->ring->num_used < (wq->ring->num_items / 2)) { 1237 sc->ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE); 1238 oce_tx_restart(sc, wq); 1239 } 1240 } 1241 } 1242 1243 1244 static void 1245 oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq) 1246 { 1247 1248 if ((sc->ifp->if_drv_flags & IFF_DRV_RUNNING) != IFF_DRV_RUNNING) 1249 return; 1250 1251 #if __FreeBSD_version >= 800000 1252 if (!drbr_empty(sc->ifp, wq->br)) 1253 #else 1254 if (!IFQ_DRV_IS_EMPTY(&sc->ifp->if_snd)) 1255 #endif 1256 taskqueue_enqueue(taskqueue_swi, &wq->txtask); 1257 1258 } 1259 1260 1261 #if defined(INET6) || defined(INET) 1262 static struct mbuf * 1263 oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp) 1264 { 1265 struct mbuf *m; 1266 #ifdef INET 1267 struct ip *ip; 1268 #endif 1269 #ifdef INET6 1270 struct ip6_hdr *ip6; 1271 #endif 1272 struct ether_vlan_header *eh; 1273 struct tcphdr *th; 1274 uint16_t etype; 1275 int total_len = 0, ehdrlen = 0; 1276 1277 m = *mpp; 1278 1279 if (M_WRITABLE(m) == 0) { 1280 m = m_dup(*mpp, M_NOWAIT); 1281 if (!m) 1282 return NULL; 1283 m_freem(*mpp); 1284 *mpp = m; 1285 } 1286 1287 eh = mtod(m, struct ether_vlan_header *); 1288 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 1289 etype = ntohs(eh->evl_proto); 1290 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 1291 } else { 1292 etype = ntohs(eh->evl_encap_proto); 1293 ehdrlen = ETHER_HDR_LEN; 1294 } 1295 1296 switch (etype) { 1297 #ifdef INET 1298 case ETHERTYPE_IP: 1299 ip = (struct ip *)(m->m_data + ehdrlen); 1300 if (ip->ip_p != IPPROTO_TCP) 1301 return NULL; 1302 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2)); 1303 1304 total_len = ehdrlen + (ip->ip_hl << 2) + (th->th_off << 2); 1305 break; 1306 #endif 1307 #ifdef INET6 1308 case ETHERTYPE_IPV6: 1309 ip6 = (struct ip6_hdr *)(m->m_data + ehdrlen); 1310 if (ip6->ip6_nxt != IPPROTO_TCP) 1311 return NULL; 1312 th = (struct tcphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr)); 1313 1314 total_len = ehdrlen + sizeof(struct ip6_hdr) + (th->th_off << 2); 1315 break; 1316 #endif 1317 default: 1318 return NULL; 1319 } 1320 1321 m = m_pullup(m, total_len); 1322 if (!m) 1323 return NULL; 1324 *mpp = m; 1325 return m; 1326 1327 } 1328 #endif /* INET6 || INET */ 1329 1330 void 1331 oce_tx_task(void *arg, int npending) 1332 { 1333 struct oce_wq *wq = arg; 1334 POCE_SOFTC sc = wq->parent; 1335 struct ifnet *ifp = sc->ifp; 1336 int rc = 0; 1337 1338 #if __FreeBSD_version >= 800000 1339 LOCK(&wq->tx_lock); 1340 rc = oce_multiq_transmit(ifp, NULL, wq); 1341 if (rc) { 1342 device_printf(sc->dev, 1343 "TX[%d] restart failed\n", wq->queue_index); 1344 } 1345 UNLOCK(&wq->tx_lock); 1346 #else 1347 oce_start(ifp); 1348 #endif 1349 1350 } 1351 1352 1353 void 1354 oce_start(struct ifnet *ifp) 1355 { 1356 POCE_SOFTC sc = ifp->if_softc; 1357 struct mbuf *m; 1358 int rc = 0; 1359 int def_q = 0; /* Defualt tx queue is 0*/ 1360 1361 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 1362 IFF_DRV_RUNNING) 1363 return; 1364 1365 if (!sc->link_status) 1366 return; 1367 1368 do { 1369 IF_DEQUEUE(&sc->ifp->if_snd, m); 1370 if (m == NULL) 1371 break; 1372 1373 LOCK(&sc->wq[def_q]->tx_lock); 1374 rc = oce_tx(sc, &m, def_q); 1375 UNLOCK(&sc->wq[def_q]->tx_lock); 1376 if (rc) { 1377 if (m != NULL) { 1378 sc->wq[def_q]->tx_stats.tx_stops ++; 1379 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1380 IFQ_DRV_PREPEND(&ifp->if_snd, m); 1381 m = NULL; 1382 } 1383 break; 1384 } 1385 if (m != NULL) 1386 ETHER_BPF_MTAP(ifp, m); 1387 1388 } while (TRUE); 1389 1390 return; 1391 } 1392 1393 1394 /* Handle the Completion Queue for transmit */ 1395 uint16_t 1396 oce_wq_handler(void *arg) 1397 { 1398 struct oce_wq *wq = (struct oce_wq *)arg; 1399 POCE_SOFTC sc = wq->parent; 1400 struct oce_cq *cq = wq->cq; 1401 struct oce_nic_tx_cqe *cqe; 1402 int num_cqes = 0; 1403 1404 LOCK(&wq->tx_compl_lock); 1405 bus_dmamap_sync(cq->ring->dma.tag, 1406 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1407 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 1408 while (cqe->u0.dw[3]) { 1409 DW_SWAP((uint32_t *) cqe, sizeof(oce_wq_cqe)); 1410 1411 wq->ring->cidx = cqe->u0.s.wqe_index + 1; 1412 if (wq->ring->cidx >= wq->ring->num_items) 1413 wq->ring->cidx -= wq->ring->num_items; 1414 1415 oce_process_tx_completion(wq); 1416 wq->tx_stats.tx_compl++; 1417 cqe->u0.dw[3] = 0; 1418 RING_GET(cq->ring, 1); 1419 bus_dmamap_sync(cq->ring->dma.tag, 1420 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1421 cqe = 1422 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 1423 num_cqes++; 1424 } 1425 1426 if (num_cqes) 1427 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); 1428 1429 UNLOCK(&wq->tx_compl_lock); 1430 return num_cqes; 1431 } 1432 1433 1434 static int 1435 oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, struct oce_wq *wq) 1436 { 1437 POCE_SOFTC sc = ifp->if_softc; 1438 int status = 0, queue_index = 0; 1439 struct mbuf *next = NULL; 1440 struct buf_ring *br = NULL; 1441 1442 br = wq->br; 1443 queue_index = wq->queue_index; 1444 1445 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 1446 IFF_DRV_RUNNING) { 1447 if (m != NULL) 1448 status = drbr_enqueue(ifp, br, m); 1449 return status; 1450 } 1451 1452 if (m != NULL) { 1453 if ((status = drbr_enqueue(ifp, br, m)) != 0) 1454 return status; 1455 } 1456 while ((next = drbr_peek(ifp, br)) != NULL) { 1457 if (oce_tx(sc, &next, queue_index)) { 1458 if (next == NULL) { 1459 drbr_advance(ifp, br); 1460 } else { 1461 drbr_putback(ifp, br, next); 1462 wq->tx_stats.tx_stops ++; 1463 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1464 } 1465 break; 1466 } 1467 drbr_advance(ifp, br); 1468 if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len); 1469 if (next->m_flags & M_MCAST) 1470 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); 1471 ETHER_BPF_MTAP(ifp, next); 1472 } 1473 1474 return 0; 1475 } 1476 1477 1478 1479 1480 /***************************************************************************** 1481 * Receive routines functions * 1482 *****************************************************************************/ 1483 1484 static void 1485 oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2) 1486 { 1487 uint32_t *p; 1488 struct ether_header *eh = NULL; 1489 struct tcphdr *tcp_hdr = NULL; 1490 struct ip *ip4_hdr = NULL; 1491 struct ip6_hdr *ip6 = NULL; 1492 uint32_t payload_len = 0; 1493 1494 eh = mtod(m, struct ether_header *); 1495 /* correct IP header */ 1496 if(!cqe2->ipv6_frame) { 1497 ip4_hdr = (struct ip *)((char*)eh + sizeof(struct ether_header)); 1498 ip4_hdr->ip_ttl = cqe2->frame_lifespan; 1499 ip4_hdr->ip_len = htons(cqe2->coalesced_size - sizeof(struct ether_header)); 1500 tcp_hdr = (struct tcphdr *)((char*)ip4_hdr + sizeof(struct ip)); 1501 }else { 1502 ip6 = (struct ip6_hdr *)((char*)eh + sizeof(struct ether_header)); 1503 ip6->ip6_ctlun.ip6_un1.ip6_un1_hlim = cqe2->frame_lifespan; 1504 payload_len = cqe2->coalesced_size - sizeof(struct ether_header) 1505 - sizeof(struct ip6_hdr); 1506 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = htons(payload_len); 1507 tcp_hdr = (struct tcphdr *)((char*)ip6 + sizeof(struct ip6_hdr)); 1508 } 1509 1510 /* correct tcp header */ 1511 tcp_hdr->th_ack = htonl(cqe2->tcp_ack_num); 1512 if(cqe2->push) { 1513 tcp_hdr->th_flags |= TH_PUSH; 1514 } 1515 tcp_hdr->th_win = htons(cqe2->tcp_window); 1516 tcp_hdr->th_sum = 0xffff; 1517 if(cqe2->ts_opt) { 1518 p = (uint32_t *)((char*)tcp_hdr + sizeof(struct tcphdr) + 2); 1519 *p = cqe1->tcp_timestamp_val; 1520 *(p+1) = cqe1->tcp_timestamp_ecr; 1521 } 1522 1523 return; 1524 } 1525 1526 static void 1527 oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m) 1528 { 1529 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1530 uint32_t i = 0, frag_len = 0; 1531 uint32_t len = cqe_info->pkt_size; 1532 struct oce_packet_desc *pd; 1533 struct mbuf *tail = NULL; 1534 1535 for (i = 0; i < cqe_info->num_frags; i++) { 1536 if (rq->ring->cidx == rq->ring->pidx) { 1537 device_printf(sc->dev, 1538 "oce_rx_mbuf_chain: Invalid RX completion - Queue is empty\n"); 1539 return; 1540 } 1541 pd = &rq->pckts[rq->ring->cidx]; 1542 1543 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1544 bus_dmamap_unload(rq->tag, pd->map); 1545 RING_GET(rq->ring, 1); 1546 rq->pending--; 1547 1548 frag_len = (len > rq->cfg.frag_size) ? rq->cfg.frag_size : len; 1549 pd->mbuf->m_len = frag_len; 1550 1551 if (tail != NULL) { 1552 /* additional fragments */ 1553 pd->mbuf->m_flags &= ~M_PKTHDR; 1554 tail->m_next = pd->mbuf; 1555 if(rq->islro) 1556 tail->m_nextpkt = NULL; 1557 tail = pd->mbuf; 1558 } else { 1559 /* first fragment, fill out much of the packet header */ 1560 pd->mbuf->m_pkthdr.len = len; 1561 if(rq->islro) 1562 pd->mbuf->m_nextpkt = NULL; 1563 pd->mbuf->m_pkthdr.csum_flags = 0; 1564 if (IF_CSUM_ENABLED(sc)) { 1565 if (cqe_info->l4_cksum_pass) { 1566 if(!cqe_info->ipv6_frame) { /* IPV4 */ 1567 pd->mbuf->m_pkthdr.csum_flags |= 1568 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1569 }else { /* IPV6 frame */ 1570 if(rq->islro) { 1571 pd->mbuf->m_pkthdr.csum_flags |= 1572 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1573 } 1574 } 1575 pd->mbuf->m_pkthdr.csum_data = 0xffff; 1576 } 1577 if (cqe_info->ip_cksum_pass) { 1578 pd->mbuf->m_pkthdr.csum_flags |= 1579 (CSUM_IP_CHECKED|CSUM_IP_VALID); 1580 } 1581 } 1582 *m = tail = pd->mbuf; 1583 } 1584 pd->mbuf = NULL; 1585 len -= frag_len; 1586 } 1587 1588 return; 1589 } 1590 1591 static void 1592 oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2) 1593 { 1594 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1595 struct nic_hwlro_cqe_part1 *cqe1 = NULL; 1596 struct mbuf *m = NULL; 1597 struct oce_common_cqe_info cq_info; 1598 1599 /* parse cqe */ 1600 if(cqe2 == NULL) { 1601 cq_info.pkt_size = cqe->pkt_size; 1602 cq_info.vtag = cqe->vlan_tag; 1603 cq_info.l4_cksum_pass = cqe->l4_cksum_pass; 1604 cq_info.ip_cksum_pass = cqe->ip_cksum_pass; 1605 cq_info.ipv6_frame = cqe->ipv6_frame; 1606 cq_info.vtp = cqe->vtp; 1607 cq_info.qnq = cqe->qnq; 1608 }else { 1609 cqe1 = (struct nic_hwlro_cqe_part1 *)cqe; 1610 cq_info.pkt_size = cqe2->coalesced_size; 1611 cq_info.vtag = cqe2->vlan_tag; 1612 cq_info.l4_cksum_pass = cqe2->l4_cksum_pass; 1613 cq_info.ip_cksum_pass = cqe2->ip_cksum_pass; 1614 cq_info.ipv6_frame = cqe2->ipv6_frame; 1615 cq_info.vtp = cqe2->vtp; 1616 cq_info.qnq = cqe1->qnq; 1617 } 1618 1619 cq_info.vtag = BSWAP_16(cq_info.vtag); 1620 1621 cq_info.num_frags = cq_info.pkt_size / rq->cfg.frag_size; 1622 if(cq_info.pkt_size % rq->cfg.frag_size) 1623 cq_info.num_frags++; 1624 1625 oce_rx_mbuf_chain(rq, &cq_info, &m); 1626 1627 if (m) { 1628 if(cqe2) { 1629 //assert(cqe2->valid != 0); 1630 1631 //assert(cqe2->cqe_type != 2); 1632 oce_correct_header(m, cqe1, cqe2); 1633 } 1634 1635 m->m_pkthdr.rcvif = sc->ifp; 1636 #if __FreeBSD_version >= 800000 1637 if (rq->queue_index) 1638 m->m_pkthdr.flowid = (rq->queue_index - 1); 1639 else 1640 m->m_pkthdr.flowid = rq->queue_index; 1641 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 1642 #endif 1643 /* This deternies if vlan tag is Valid */ 1644 if (cq_info.vtp) { 1645 if (sc->function_mode & FNM_FLEX10_MODE) { 1646 /* FLEX10. If QnQ is not set, neglect VLAN */ 1647 if (cq_info.qnq) { 1648 m->m_pkthdr.ether_vtag = cq_info.vtag; 1649 m->m_flags |= M_VLANTAG; 1650 } 1651 } else if (sc->pvid != (cq_info.vtag & VLAN_VID_MASK)) { 1652 /* In UMC mode generally pvid will be striped by 1653 hw. But in some cases we have seen it comes 1654 with pvid. So if pvid == vlan, neglect vlan. 1655 */ 1656 m->m_pkthdr.ether_vtag = cq_info.vtag; 1657 m->m_flags |= M_VLANTAG; 1658 } 1659 } 1660 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 1661 1662 (*sc->ifp->if_input) (sc->ifp, m); 1663 1664 /* Update rx stats per queue */ 1665 rq->rx_stats.rx_pkts++; 1666 rq->rx_stats.rx_bytes += cq_info.pkt_size; 1667 rq->rx_stats.rx_frags += cq_info.num_frags; 1668 rq->rx_stats.rx_ucast_pkts++; 1669 } 1670 return; 1671 } 1672 1673 static void 1674 oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe) 1675 { 1676 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1677 int len; 1678 struct mbuf *m = NULL; 1679 struct oce_common_cqe_info cq_info; 1680 uint16_t vtag = 0; 1681 1682 /* Is it a flush compl that has no data */ 1683 if(!cqe->u0.s.num_fragments) 1684 goto exit; 1685 1686 len = cqe->u0.s.pkt_size; 1687 if (!len) { 1688 /*partial DMA workaround for Lancer*/ 1689 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); 1690 goto exit; 1691 } 1692 1693 if (!oce_cqe_portid_valid(sc, cqe)) { 1694 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); 1695 goto exit; 1696 } 1697 1698 /* Get vlan_tag value */ 1699 if(IS_BE(sc) || IS_SH(sc)) 1700 vtag = BSWAP_16(cqe->u0.s.vlan_tag); 1701 else 1702 vtag = cqe->u0.s.vlan_tag; 1703 1704 cq_info.l4_cksum_pass = cqe->u0.s.l4_cksum_pass; 1705 cq_info.ip_cksum_pass = cqe->u0.s.ip_cksum_pass; 1706 cq_info.ipv6_frame = cqe->u0.s.ip_ver; 1707 cq_info.num_frags = cqe->u0.s.num_fragments; 1708 cq_info.pkt_size = cqe->u0.s.pkt_size; 1709 1710 oce_rx_mbuf_chain(rq, &cq_info, &m); 1711 1712 if (m) { 1713 m->m_pkthdr.rcvif = sc->ifp; 1714 #if __FreeBSD_version >= 800000 1715 if (rq->queue_index) 1716 m->m_pkthdr.flowid = (rq->queue_index - 1); 1717 else 1718 m->m_pkthdr.flowid = rq->queue_index; 1719 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 1720 #endif 1721 /* This deternies if vlan tag is Valid */ 1722 if (oce_cqe_vtp_valid(sc, cqe)) { 1723 if (sc->function_mode & FNM_FLEX10_MODE) { 1724 /* FLEX10. If QnQ is not set, neglect VLAN */ 1725 if (cqe->u0.s.qnq) { 1726 m->m_pkthdr.ether_vtag = vtag; 1727 m->m_flags |= M_VLANTAG; 1728 } 1729 } else if (sc->pvid != (vtag & VLAN_VID_MASK)) { 1730 /* In UMC mode generally pvid will be striped by 1731 hw. But in some cases we have seen it comes 1732 with pvid. So if pvid == vlan, neglect vlan. 1733 */ 1734 m->m_pkthdr.ether_vtag = vtag; 1735 m->m_flags |= M_VLANTAG; 1736 } 1737 } 1738 1739 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 1740 #if defined(INET6) || defined(INET) 1741 /* Try to queue to LRO */ 1742 if (IF_LRO_ENABLED(sc) && 1743 (cqe->u0.s.ip_cksum_pass) && 1744 (cqe->u0.s.l4_cksum_pass) && 1745 (!cqe->u0.s.ip_ver) && 1746 (rq->lro.lro_cnt != 0)) { 1747 1748 if (tcp_lro_rx(&rq->lro, m, 0) == 0) { 1749 rq->lro_pkts_queued ++; 1750 goto post_done; 1751 } 1752 /* If LRO posting fails then try to post to STACK */ 1753 } 1754 #endif 1755 1756 (*sc->ifp->if_input) (sc->ifp, m); 1757 #if defined(INET6) || defined(INET) 1758 post_done: 1759 #endif 1760 /* Update rx stats per queue */ 1761 rq->rx_stats.rx_pkts++; 1762 rq->rx_stats.rx_bytes += cqe->u0.s.pkt_size; 1763 rq->rx_stats.rx_frags += cqe->u0.s.num_fragments; 1764 if (cqe->u0.s.pkt_type == OCE_MULTICAST_PACKET) 1765 rq->rx_stats.rx_mcast_pkts++; 1766 if (cqe->u0.s.pkt_type == OCE_UNICAST_PACKET) 1767 rq->rx_stats.rx_ucast_pkts++; 1768 } 1769 exit: 1770 return; 1771 } 1772 1773 1774 void 1775 oce_discard_rx_comp(struct oce_rq *rq, int num_frags) 1776 { 1777 uint32_t i = 0; 1778 struct oce_packet_desc *pd; 1779 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1780 1781 for (i = 0; i < num_frags; i++) { 1782 if (rq->ring->cidx == rq->ring->pidx) { 1783 device_printf(sc->dev, 1784 "oce_discard_rx_comp: Invalid RX completion - Queue is empty\n"); 1785 return; 1786 } 1787 pd = &rq->pckts[rq->ring->cidx]; 1788 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1789 bus_dmamap_unload(rq->tag, pd->map); 1790 if (pd->mbuf != NULL) { 1791 m_freem(pd->mbuf); 1792 pd->mbuf = NULL; 1793 } 1794 1795 RING_GET(rq->ring, 1); 1796 rq->pending--; 1797 } 1798 } 1799 1800 1801 static int 1802 oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe) 1803 { 1804 struct oce_nic_rx_cqe_v1 *cqe_v1; 1805 int vtp = 0; 1806 1807 if (sc->be3_native) { 1808 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe; 1809 vtp = cqe_v1->u0.s.vlan_tag_present; 1810 } else 1811 vtp = cqe->u0.s.vlan_tag_present; 1812 1813 return vtp; 1814 1815 } 1816 1817 1818 static int 1819 oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe) 1820 { 1821 struct oce_nic_rx_cqe_v1 *cqe_v1; 1822 int port_id = 0; 1823 1824 if (sc->be3_native && (IS_BE(sc) || IS_SH(sc))) { 1825 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe; 1826 port_id = cqe_v1->u0.s.port; 1827 if (sc->port_id != port_id) 1828 return 0; 1829 } else 1830 ;/* For BE3 legacy and Lancer this is dummy */ 1831 1832 return 1; 1833 1834 } 1835 1836 #if defined(INET6) || defined(INET) 1837 void 1838 oce_rx_flush_lro(struct oce_rq *rq) 1839 { 1840 struct lro_ctrl *lro = &rq->lro; 1841 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1842 1843 if (!IF_LRO_ENABLED(sc)) 1844 return; 1845 1846 tcp_lro_flush_all(lro); 1847 rq->lro_pkts_queued = 0; 1848 1849 return; 1850 } 1851 1852 1853 static int 1854 oce_init_lro(POCE_SOFTC sc) 1855 { 1856 struct lro_ctrl *lro = NULL; 1857 int i = 0, rc = 0; 1858 1859 for (i = 0; i < sc->nrqs; i++) { 1860 lro = &sc->rq[i]->lro; 1861 rc = tcp_lro_init(lro); 1862 if (rc != 0) { 1863 device_printf(sc->dev, "LRO init failed\n"); 1864 return rc; 1865 } 1866 lro->ifp = sc->ifp; 1867 } 1868 1869 return rc; 1870 } 1871 1872 1873 void 1874 oce_free_lro(POCE_SOFTC sc) 1875 { 1876 struct lro_ctrl *lro = NULL; 1877 int i = 0; 1878 1879 for (i = 0; i < sc->nrqs; i++) { 1880 lro = &sc->rq[i]->lro; 1881 if (lro) 1882 tcp_lro_free(lro); 1883 } 1884 } 1885 #endif 1886 1887 int 1888 oce_alloc_rx_bufs(struct oce_rq *rq, int count) 1889 { 1890 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1891 int i, in, rc; 1892 struct oce_packet_desc *pd; 1893 bus_dma_segment_t segs[6]; 1894 int nsegs, added = 0; 1895 struct oce_nic_rqe *rqe; 1896 pd_rxulp_db_t rxdb_reg; 1897 uint32_t val = 0; 1898 uint32_t oce_max_rq_posts = 64; 1899 1900 bzero(&rxdb_reg, sizeof(pd_rxulp_db_t)); 1901 for (i = 0; i < count; i++) { 1902 in = (rq->ring->pidx + 1) % OCE_RQ_PACKET_ARRAY_SIZE; 1903 1904 pd = &rq->pckts[rq->ring->pidx]; 1905 pd->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, oce_rq_buf_size); 1906 if (pd->mbuf == NULL) { 1907 device_printf(sc->dev, "mbuf allocation failed, size = %d\n",oce_rq_buf_size); 1908 break; 1909 } 1910 pd->mbuf->m_nextpkt = NULL; 1911 1912 pd->mbuf->m_len = pd->mbuf->m_pkthdr.len = rq->cfg.frag_size; 1913 1914 rc = bus_dmamap_load_mbuf_sg(rq->tag, 1915 pd->map, 1916 pd->mbuf, 1917 segs, &nsegs, BUS_DMA_NOWAIT); 1918 if (rc) { 1919 m_free(pd->mbuf); 1920 device_printf(sc->dev, "bus_dmamap_load_mbuf_sg failed rc = %d\n", rc); 1921 break; 1922 } 1923 1924 if (nsegs != 1) { 1925 i--; 1926 continue; 1927 } 1928 1929 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_PREREAD); 1930 1931 rqe = RING_GET_PRODUCER_ITEM_VA(rq->ring, struct oce_nic_rqe); 1932 rqe->u0.s.frag_pa_hi = ADDR_HI(segs[0].ds_addr); 1933 rqe->u0.s.frag_pa_lo = ADDR_LO(segs[0].ds_addr); 1934 DW_SWAP(u32ptr(rqe), sizeof(struct oce_nic_rqe)); 1935 RING_PUT(rq->ring, 1); 1936 added++; 1937 rq->pending++; 1938 } 1939 oce_max_rq_posts = sc->enable_hwlro ? OCE_HWLRO_MAX_RQ_POSTS : OCE_MAX_RQ_POSTS; 1940 if (added != 0) { 1941 for (i = added / oce_max_rq_posts; i > 0; i--) { 1942 rxdb_reg.bits.num_posted = oce_max_rq_posts; 1943 rxdb_reg.bits.qid = rq->rq_id; 1944 if(rq->islro) { 1945 val |= rq->rq_id & DB_LRO_RQ_ID_MASK; 1946 val |= oce_max_rq_posts << 16; 1947 OCE_WRITE_REG32(sc, db, DB_OFFSET, val); 1948 }else { 1949 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); 1950 } 1951 added -= oce_max_rq_posts; 1952 } 1953 if (added > 0) { 1954 rxdb_reg.bits.qid = rq->rq_id; 1955 rxdb_reg.bits.num_posted = added; 1956 if(rq->islro) { 1957 val |= rq->rq_id & DB_LRO_RQ_ID_MASK; 1958 val |= added << 16; 1959 OCE_WRITE_REG32(sc, db, DB_OFFSET, val); 1960 }else { 1961 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); 1962 } 1963 } 1964 } 1965 1966 return 0; 1967 } 1968 1969 static void 1970 oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq) 1971 { 1972 if (num_cqes) { 1973 oce_arm_cq(sc, rq->cq->cq_id, num_cqes, FALSE); 1974 if(!sc->enable_hwlro) { 1975 if((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) > 1) 1976 oce_alloc_rx_bufs(rq, ((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) - 1)); 1977 }else { 1978 if ((OCE_RQ_PACKET_ARRAY_SIZE -1 - rq->pending) > 64) 1979 oce_alloc_rx_bufs(rq, 64); 1980 } 1981 } 1982 1983 return; 1984 } 1985 1986 uint16_t 1987 oce_rq_handler_lro(void *arg) 1988 { 1989 struct oce_rq *rq = (struct oce_rq *)arg; 1990 struct oce_cq *cq = rq->cq; 1991 POCE_SOFTC sc = rq->parent; 1992 struct nic_hwlro_singleton_cqe *cqe; 1993 struct nic_hwlro_cqe_part2 *cqe2; 1994 int num_cqes = 0; 1995 1996 LOCK(&rq->rx_lock); 1997 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1998 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); 1999 while (cqe->valid) { 2000 if(cqe->cqe_type == 0) { /* singleton cqe */ 2001 /* we should not get singleton cqe after cqe1 on same rq */ 2002 if(rq->cqe_firstpart != NULL) { 2003 device_printf(sc->dev, "Got singleton cqe after cqe1 \n"); 2004 goto exit_rq_handler_lro; 2005 } 2006 if(cqe->error != 0) { 2007 rq->rx_stats.rxcp_err++; 2008 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2009 } 2010 oce_rx_lro(rq, cqe, NULL); 2011 rq->rx_stats.rx_compl++; 2012 cqe->valid = 0; 2013 RING_GET(cq->ring, 1); 2014 num_cqes++; 2015 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2016 break; 2017 }else if(cqe->cqe_type == 0x1) { /* first part */ 2018 /* we should not get cqe1 after cqe1 on same rq */ 2019 if(rq->cqe_firstpart != NULL) { 2020 device_printf(sc->dev, "Got cqe1 after cqe1 \n"); 2021 goto exit_rq_handler_lro; 2022 } 2023 rq->cqe_firstpart = (struct nic_hwlro_cqe_part1 *)cqe; 2024 RING_GET(cq->ring, 1); 2025 }else if(cqe->cqe_type == 0x2) { /* second part */ 2026 cqe2 = (struct nic_hwlro_cqe_part2 *)cqe; 2027 if(cqe2->error != 0) { 2028 rq->rx_stats.rxcp_err++; 2029 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2030 } 2031 /* We should not get cqe2 without cqe1 */ 2032 if(rq->cqe_firstpart == NULL) { 2033 device_printf(sc->dev, "Got cqe2 without cqe1 \n"); 2034 goto exit_rq_handler_lro; 2035 } 2036 oce_rx_lro(rq, (struct nic_hwlro_singleton_cqe *)rq->cqe_firstpart, cqe2); 2037 2038 rq->rx_stats.rx_compl++; 2039 rq->cqe_firstpart->valid = 0; 2040 cqe2->valid = 0; 2041 rq->cqe_firstpart = NULL; 2042 2043 RING_GET(cq->ring, 1); 2044 num_cqes += 2; 2045 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2046 break; 2047 } 2048 2049 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2050 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); 2051 } 2052 oce_check_rx_bufs(sc, num_cqes, rq); 2053 exit_rq_handler_lro: 2054 UNLOCK(&rq->rx_lock); 2055 return 0; 2056 } 2057 2058 /* Handle the Completion Queue for receive */ 2059 uint16_t 2060 oce_rq_handler(void *arg) 2061 { 2062 struct oce_rq *rq = (struct oce_rq *)arg; 2063 struct oce_cq *cq = rq->cq; 2064 POCE_SOFTC sc = rq->parent; 2065 struct oce_nic_rx_cqe *cqe; 2066 int num_cqes = 0; 2067 2068 if(rq->islro) { 2069 oce_rq_handler_lro(arg); 2070 return 0; 2071 } 2072 LOCK(&rq->rx_lock); 2073 bus_dmamap_sync(cq->ring->dma.tag, 2074 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2075 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); 2076 while (cqe->u0.dw[2]) { 2077 DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe)); 2078 2079 if (cqe->u0.s.error == 0) { 2080 oce_rx(rq, cqe); 2081 } else { 2082 rq->rx_stats.rxcp_err++; 2083 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2084 /* Post L3/L4 errors to stack.*/ 2085 oce_rx(rq, cqe); 2086 } 2087 rq->rx_stats.rx_compl++; 2088 cqe->u0.dw[2] = 0; 2089 2090 #if defined(INET6) || defined(INET) 2091 if (IF_LRO_ENABLED(sc) && rq->lro_pkts_queued >= 16) { 2092 oce_rx_flush_lro(rq); 2093 } 2094 #endif 2095 2096 RING_GET(cq->ring, 1); 2097 bus_dmamap_sync(cq->ring->dma.tag, 2098 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2099 cqe = 2100 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); 2101 num_cqes++; 2102 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2103 break; 2104 } 2105 2106 #if defined(INET6) || defined(INET) 2107 if (IF_LRO_ENABLED(sc)) 2108 oce_rx_flush_lro(rq); 2109 #endif 2110 2111 oce_check_rx_bufs(sc, num_cqes, rq); 2112 UNLOCK(&rq->rx_lock); 2113 return 0; 2114 2115 } 2116 2117 2118 2119 2120 /***************************************************************************** 2121 * Helper function prototypes in this file * 2122 *****************************************************************************/ 2123 2124 static int 2125 oce_attach_ifp(POCE_SOFTC sc) 2126 { 2127 2128 sc->ifp = if_alloc(IFT_ETHER); 2129 if (!sc->ifp) 2130 return ENOMEM; 2131 2132 ifmedia_init(&sc->media, IFM_IMASK, oce_media_change, oce_media_status); 2133 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2134 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 2135 2136 sc->ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST; 2137 sc->ifp->if_ioctl = oce_ioctl; 2138 sc->ifp->if_start = oce_start; 2139 sc->ifp->if_init = oce_init; 2140 sc->ifp->if_mtu = ETHERMTU; 2141 sc->ifp->if_softc = sc; 2142 #if __FreeBSD_version >= 800000 2143 sc->ifp->if_transmit = oce_multiq_start; 2144 sc->ifp->if_qflush = oce_multiq_flush; 2145 #endif 2146 2147 if_initname(sc->ifp, 2148 device_get_name(sc->dev), device_get_unit(sc->dev)); 2149 2150 sc->ifp->if_snd.ifq_drv_maxlen = OCE_MAX_TX_DESC - 1; 2151 IFQ_SET_MAXLEN(&sc->ifp->if_snd, sc->ifp->if_snd.ifq_drv_maxlen); 2152 IFQ_SET_READY(&sc->ifp->if_snd); 2153 2154 sc->ifp->if_hwassist = OCE_IF_HWASSIST; 2155 sc->ifp->if_hwassist |= CSUM_TSO; 2156 sc->ifp->if_hwassist |= (CSUM_IP | CSUM_TCP | CSUM_UDP); 2157 2158 sc->ifp->if_capabilities = OCE_IF_CAPABILITIES; 2159 sc->ifp->if_capabilities |= IFCAP_HWCSUM; 2160 sc->ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 2161 2162 #if defined(INET6) || defined(INET) 2163 sc->ifp->if_capabilities |= IFCAP_TSO; 2164 sc->ifp->if_capabilities |= IFCAP_LRO; 2165 sc->ifp->if_capabilities |= IFCAP_VLAN_HWTSO; 2166 #endif 2167 2168 sc->ifp->if_capenable = sc->ifp->if_capabilities; 2169 sc->ifp->if_baudrate = IF_Gbps(10); 2170 2171 #if __FreeBSD_version >= 1000000 2172 sc->ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 2173 sc->ifp->if_hw_tsomaxsegcount = OCE_MAX_TX_ELEMENTS; 2174 sc->ifp->if_hw_tsomaxsegsize = 4096; 2175 #endif 2176 2177 ether_ifattach(sc->ifp, sc->macaddr.mac_addr); 2178 2179 return 0; 2180 } 2181 2182 2183 static void 2184 oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag) 2185 { 2186 POCE_SOFTC sc = ifp->if_softc; 2187 2188 if (ifp->if_softc != arg) 2189 return; 2190 if ((vtag == 0) || (vtag > 4095)) 2191 return; 2192 2193 sc->vlan_tag[vtag] = 1; 2194 sc->vlans_added++; 2195 if (sc->vlans_added <= (sc->max_vlans + 1)) 2196 oce_vid_config(sc); 2197 } 2198 2199 2200 static void 2201 oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag) 2202 { 2203 POCE_SOFTC sc = ifp->if_softc; 2204 2205 if (ifp->if_softc != arg) 2206 return; 2207 if ((vtag == 0) || (vtag > 4095)) 2208 return; 2209 2210 sc->vlan_tag[vtag] = 0; 2211 sc->vlans_added--; 2212 oce_vid_config(sc); 2213 } 2214 2215 2216 /* 2217 * A max of 64 vlans can be configured in BE. If the user configures 2218 * more, place the card in vlan promiscuous mode. 2219 */ 2220 static int 2221 oce_vid_config(POCE_SOFTC sc) 2222 { 2223 struct normal_vlan vtags[MAX_VLANFILTER_SIZE]; 2224 uint16_t ntags = 0, i; 2225 int status = 0; 2226 2227 if ((sc->vlans_added <= MAX_VLANFILTER_SIZE) && 2228 (sc->ifp->if_capenable & IFCAP_VLAN_HWFILTER)) { 2229 for (i = 0; i < MAX_VLANS; i++) { 2230 if (sc->vlan_tag[i]) { 2231 vtags[ntags].vtag = i; 2232 ntags++; 2233 } 2234 } 2235 if (ntags) 2236 status = oce_config_vlan(sc, (uint8_t) sc->if_id, 2237 vtags, ntags, 1, 0); 2238 } else 2239 status = oce_config_vlan(sc, (uint8_t) sc->if_id, 2240 NULL, 0, 1, 1); 2241 return status; 2242 } 2243 2244 2245 static void 2246 oce_mac_addr_set(POCE_SOFTC sc) 2247 { 2248 uint32_t old_pmac_id = sc->pmac_id; 2249 int status = 0; 2250 2251 2252 status = bcmp((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr, 2253 sc->macaddr.size_of_struct); 2254 if (!status) 2255 return; 2256 2257 status = oce_mbox_macaddr_add(sc, (uint8_t *)(IF_LLADDR(sc->ifp)), 2258 sc->if_id, &sc->pmac_id); 2259 if (!status) { 2260 status = oce_mbox_macaddr_del(sc, sc->if_id, old_pmac_id); 2261 bcopy((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr, 2262 sc->macaddr.size_of_struct); 2263 } 2264 if (status) 2265 device_printf(sc->dev, "Failed update macaddress\n"); 2266 2267 } 2268 2269 2270 static int 2271 oce_handle_passthrough(struct ifnet *ifp, caddr_t data) 2272 { 2273 POCE_SOFTC sc = ifp->if_softc; 2274 struct ifreq *ifr = (struct ifreq *)data; 2275 int rc = ENXIO; 2276 char cookie[32] = {0}; 2277 void *priv_data = (void *)ifr->ifr_data; 2278 void *ioctl_ptr; 2279 uint32_t req_size; 2280 struct mbx_hdr req; 2281 OCE_DMA_MEM dma_mem; 2282 struct mbx_common_get_cntl_attr *fw_cmd; 2283 2284 if (copyin(priv_data, cookie, strlen(IOCTL_COOKIE))) 2285 return EFAULT; 2286 2287 if (memcmp(cookie, IOCTL_COOKIE, strlen(IOCTL_COOKIE))) 2288 return EINVAL; 2289 2290 ioctl_ptr = (char *)priv_data + strlen(IOCTL_COOKIE); 2291 if (copyin(ioctl_ptr, &req, sizeof(struct mbx_hdr))) 2292 return EFAULT; 2293 2294 req_size = le32toh(req.u0.req.request_length); 2295 if (req_size > 65536) 2296 return EINVAL; 2297 2298 req_size += sizeof(struct mbx_hdr); 2299 rc = oce_dma_alloc(sc, req_size, &dma_mem, 0); 2300 if (rc) 2301 return ENOMEM; 2302 2303 if (copyin(ioctl_ptr, OCE_DMAPTR(&dma_mem,char), req_size)) { 2304 rc = EFAULT; 2305 goto dma_free; 2306 } 2307 2308 rc = oce_pass_through_mbox(sc, &dma_mem, req_size); 2309 if (rc) { 2310 rc = EIO; 2311 goto dma_free; 2312 } 2313 2314 if (copyout(OCE_DMAPTR(&dma_mem,char), ioctl_ptr, req_size)) 2315 rc = EFAULT; 2316 2317 /* 2318 firmware is filling all the attributes for this ioctl except 2319 the driver version..so fill it 2320 */ 2321 if(req.u0.rsp.opcode == OPCODE_COMMON_GET_CNTL_ATTRIBUTES) { 2322 fw_cmd = (struct mbx_common_get_cntl_attr *) ioctl_ptr; 2323 strncpy(fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str, 2324 COMPONENT_REVISION, strlen(COMPONENT_REVISION)); 2325 } 2326 2327 dma_free: 2328 oce_dma_free(sc, &dma_mem); 2329 return rc; 2330 2331 } 2332 2333 static void 2334 oce_eqd_set_periodic(POCE_SOFTC sc) 2335 { 2336 struct oce_set_eqd set_eqd[OCE_MAX_EQ]; 2337 struct oce_aic_obj *aic; 2338 struct oce_eq *eqo; 2339 uint64_t now = 0, delta; 2340 int eqd, i, num = 0; 2341 uint32_t tx_reqs = 0, rxpkts = 0, pps; 2342 struct oce_wq *wq; 2343 struct oce_rq *rq; 2344 2345 #define ticks_to_msecs(t) (1000 * (t) / hz) 2346 2347 for (i = 0 ; i < sc->neqs; i++) { 2348 eqo = sc->eq[i]; 2349 aic = &sc->aic_obj[i]; 2350 /* When setting the static eq delay from the user space */ 2351 if (!aic->enable) { 2352 if (aic->ticks) 2353 aic->ticks = 0; 2354 eqd = aic->et_eqd; 2355 goto modify_eqd; 2356 } 2357 2358 rq = sc->rq[i]; 2359 rxpkts = rq->rx_stats.rx_pkts; 2360 wq = sc->wq[i]; 2361 tx_reqs = wq->tx_stats.tx_reqs; 2362 now = ticks; 2363 2364 if (!aic->ticks || now < aic->ticks || 2365 rxpkts < aic->prev_rxpkts || tx_reqs < aic->prev_txreqs) { 2366 aic->prev_rxpkts = rxpkts; 2367 aic->prev_txreqs = tx_reqs; 2368 aic->ticks = now; 2369 continue; 2370 } 2371 2372 delta = ticks_to_msecs(now - aic->ticks); 2373 2374 pps = (((uint32_t)(rxpkts - aic->prev_rxpkts) * 1000) / delta) + 2375 (((uint32_t)(tx_reqs - aic->prev_txreqs) * 1000) / delta); 2376 eqd = (pps / 15000) << 2; 2377 if (eqd < 8) 2378 eqd = 0; 2379 2380 /* Make sure that the eq delay is in the known range */ 2381 eqd = min(eqd, aic->max_eqd); 2382 eqd = max(eqd, aic->min_eqd); 2383 2384 aic->prev_rxpkts = rxpkts; 2385 aic->prev_txreqs = tx_reqs; 2386 aic->ticks = now; 2387 2388 modify_eqd: 2389 if (eqd != aic->cur_eqd) { 2390 set_eqd[num].delay_multiplier = (eqd * 65)/100; 2391 set_eqd[num].eq_id = eqo->eq_id; 2392 aic->cur_eqd = eqd; 2393 num++; 2394 } 2395 } 2396 2397 /* Is there atleast one eq that needs to be modified? */ 2398 for(i = 0; i < num; i += 8) { 2399 if((num - i) >=8 ) 2400 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], 8); 2401 else 2402 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], (num - i)); 2403 } 2404 2405 } 2406 2407 static void oce_detect_hw_error(POCE_SOFTC sc) 2408 { 2409 2410 uint32_t ue_low = 0, ue_high = 0, ue_low_mask = 0, ue_high_mask = 0; 2411 uint32_t sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0; 2412 uint32_t i; 2413 2414 if (sc->hw_error) 2415 return; 2416 2417 if (IS_XE201(sc)) { 2418 sliport_status = OCE_READ_REG32(sc, db, SLIPORT_STATUS_OFFSET); 2419 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2420 sliport_err1 = OCE_READ_REG32(sc, db, SLIPORT_ERROR1_OFFSET); 2421 sliport_err2 = OCE_READ_REG32(sc, db, SLIPORT_ERROR2_OFFSET); 2422 } 2423 } else { 2424 ue_low = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW); 2425 ue_high = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HIGH); 2426 ue_low_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW_MASK); 2427 ue_high_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HI_MASK); 2428 2429 ue_low = (ue_low & ~ue_low_mask); 2430 ue_high = (ue_high & ~ue_high_mask); 2431 } 2432 2433 /* On certain platforms BE hardware can indicate spurious UEs. 2434 * Allow the h/w to stop working completely in case of a real UE. 2435 * Hence not setting the hw_error for UE detection. 2436 */ 2437 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2438 sc->hw_error = TRUE; 2439 device_printf(sc->dev, "Error detected in the card\n"); 2440 } 2441 2442 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2443 device_printf(sc->dev, 2444 "ERR: sliport status 0x%x\n", sliport_status); 2445 device_printf(sc->dev, 2446 "ERR: sliport error1 0x%x\n", sliport_err1); 2447 device_printf(sc->dev, 2448 "ERR: sliport error2 0x%x\n", sliport_err2); 2449 } 2450 2451 if (ue_low) { 2452 for (i = 0; ue_low; ue_low >>= 1, i++) { 2453 if (ue_low & 1) 2454 device_printf(sc->dev, "UE: %s bit set\n", 2455 ue_status_low_desc[i]); 2456 } 2457 } 2458 2459 if (ue_high) { 2460 for (i = 0; ue_high; ue_high >>= 1, i++) { 2461 if (ue_high & 1) 2462 device_printf(sc->dev, "UE: %s bit set\n", 2463 ue_status_hi_desc[i]); 2464 } 2465 } 2466 2467 } 2468 2469 2470 static void 2471 oce_local_timer(void *arg) 2472 { 2473 POCE_SOFTC sc = arg; 2474 int i = 0; 2475 2476 oce_detect_hw_error(sc); 2477 oce_refresh_nic_stats(sc); 2478 oce_refresh_queue_stats(sc); 2479 oce_mac_addr_set(sc); 2480 2481 /* TX Watch Dog*/ 2482 for (i = 0; i < sc->nwqs; i++) 2483 oce_tx_restart(sc, sc->wq[i]); 2484 2485 /* calculate and set the eq delay for optimal interrupt rate */ 2486 if (IS_BE(sc) || IS_SH(sc)) 2487 oce_eqd_set_periodic(sc); 2488 2489 callout_reset(&sc->timer, hz, oce_local_timer, sc); 2490 } 2491 2492 static void 2493 oce_tx_compl_clean(POCE_SOFTC sc) 2494 { 2495 struct oce_wq *wq; 2496 int i = 0, timeo = 0, num_wqes = 0; 2497 int pending_txqs = sc->nwqs; 2498 2499 /* Stop polling for compls when HW has been silent for 10ms or 2500 * hw_error or no outstanding completions expected 2501 */ 2502 do { 2503 pending_txqs = sc->nwqs; 2504 2505 for_all_wq_queues(sc, wq, i) { 2506 num_wqes = oce_wq_handler(wq); 2507 2508 if(num_wqes) 2509 timeo = 0; 2510 2511 if(!wq->ring->num_used) 2512 pending_txqs--; 2513 } 2514 2515 if (pending_txqs == 0 || ++timeo > 10 || sc->hw_error) 2516 break; 2517 2518 DELAY(1000); 2519 } while (TRUE); 2520 2521 for_all_wq_queues(sc, wq, i) { 2522 while(wq->ring->num_used) { 2523 LOCK(&wq->tx_compl_lock); 2524 oce_process_tx_completion(wq); 2525 UNLOCK(&wq->tx_compl_lock); 2526 } 2527 } 2528 2529 } 2530 2531 /* NOTE : This should only be called holding 2532 * DEVICE_LOCK. 2533 */ 2534 static void 2535 oce_if_deactivate(POCE_SOFTC sc) 2536 { 2537 int i; 2538 struct oce_rq *rq; 2539 struct oce_wq *wq; 2540 struct oce_eq *eq; 2541 2542 sc->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 2543 2544 oce_tx_compl_clean(sc); 2545 2546 /* Stop intrs and finish any bottom halves pending */ 2547 oce_hw_intr_disable(sc); 2548 2549 /* Since taskqueue_drain takes a Gaint Lock, We should not acquire 2550 any other lock. So unlock device lock and require after 2551 completing taskqueue_drain. 2552 */ 2553 UNLOCK(&sc->dev_lock); 2554 for (i = 0; i < sc->intr_count; i++) { 2555 if (sc->intrs[i].tq != NULL) { 2556 taskqueue_drain(sc->intrs[i].tq, &sc->intrs[i].task); 2557 } 2558 } 2559 LOCK(&sc->dev_lock); 2560 2561 /* Delete RX queue in card with flush param */ 2562 oce_stop_rx(sc); 2563 2564 /* Invalidate any pending cq and eq entries*/ 2565 for_all_evnt_queues(sc, eq, i) 2566 oce_drain_eq(eq); 2567 for_all_rq_queues(sc, rq, i) 2568 oce_drain_rq_cq(rq); 2569 for_all_wq_queues(sc, wq, i) 2570 oce_drain_wq_cq(wq); 2571 2572 /* But still we need to get MCC aync events. 2573 So enable intrs and also arm first EQ 2574 */ 2575 oce_hw_intr_enable(sc); 2576 oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE); 2577 2578 DELAY(10); 2579 } 2580 2581 2582 static void 2583 oce_if_activate(POCE_SOFTC sc) 2584 { 2585 struct oce_eq *eq; 2586 struct oce_rq *rq; 2587 struct oce_wq *wq; 2588 int i, rc = 0; 2589 2590 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; 2591 2592 oce_hw_intr_disable(sc); 2593 2594 oce_start_rx(sc); 2595 2596 for_all_rq_queues(sc, rq, i) { 2597 rc = oce_start_rq(rq); 2598 if (rc) 2599 device_printf(sc->dev, "Unable to start RX\n"); 2600 } 2601 2602 for_all_wq_queues(sc, wq, i) { 2603 rc = oce_start_wq(wq); 2604 if (rc) 2605 device_printf(sc->dev, "Unable to start TX\n"); 2606 } 2607 2608 2609 for_all_evnt_queues(sc, eq, i) 2610 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); 2611 2612 oce_hw_intr_enable(sc); 2613 2614 } 2615 2616 static void 2617 process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe) 2618 { 2619 /* Update Link status */ 2620 if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) == 2621 ASYNC_EVENT_LINK_UP) { 2622 sc->link_status = ASYNC_EVENT_LINK_UP; 2623 if_link_state_change(sc->ifp, LINK_STATE_UP); 2624 } else { 2625 sc->link_status = ASYNC_EVENT_LINK_DOWN; 2626 if_link_state_change(sc->ifp, LINK_STATE_DOWN); 2627 } 2628 } 2629 2630 2631 static void oce_async_grp5_osbmc_process(POCE_SOFTC sc, 2632 struct oce_async_evt_grp5_os2bmc *evt) 2633 { 2634 DW_SWAP(evt, sizeof(struct oce_async_evt_grp5_os2bmc)); 2635 if (evt->u.s.mgmt_enable) 2636 sc->flags |= OCE_FLAGS_OS2BMC; 2637 else 2638 return; 2639 2640 sc->bmc_filt_mask = evt->u.s.arp_filter; 2641 sc->bmc_filt_mask |= (evt->u.s.dhcp_client_filt << 1); 2642 sc->bmc_filt_mask |= (evt->u.s.dhcp_server_filt << 2); 2643 sc->bmc_filt_mask |= (evt->u.s.net_bios_filt << 3); 2644 sc->bmc_filt_mask |= (evt->u.s.bcast_filt << 4); 2645 sc->bmc_filt_mask |= (evt->u.s.ipv6_nbr_filt << 5); 2646 sc->bmc_filt_mask |= (evt->u.s.ipv6_ra_filt << 6); 2647 sc->bmc_filt_mask |= (evt->u.s.ipv6_ras_filt << 7); 2648 sc->bmc_filt_mask |= (evt->u.s.mcast_filt << 8); 2649 } 2650 2651 2652 static void oce_process_grp5_events(POCE_SOFTC sc, struct oce_mq_cqe *cqe) 2653 { 2654 struct oce_async_event_grp5_pvid_state *gcqe; 2655 struct oce_async_evt_grp5_os2bmc *bmccqe; 2656 2657 switch (cqe->u0.s.async_type) { 2658 case ASYNC_EVENT_PVID_STATE: 2659 /* GRP5 PVID */ 2660 gcqe = (struct oce_async_event_grp5_pvid_state *)cqe; 2661 if (gcqe->enabled) 2662 sc->pvid = gcqe->tag & VLAN_VID_MASK; 2663 else 2664 sc->pvid = 0; 2665 break; 2666 case ASYNC_EVENT_OS2BMC: 2667 bmccqe = (struct oce_async_evt_grp5_os2bmc *)cqe; 2668 oce_async_grp5_osbmc_process(sc, bmccqe); 2669 break; 2670 default: 2671 break; 2672 } 2673 } 2674 2675 /* Handle the Completion Queue for the Mailbox/Async notifications */ 2676 uint16_t 2677 oce_mq_handler(void *arg) 2678 { 2679 struct oce_mq *mq = (struct oce_mq *)arg; 2680 POCE_SOFTC sc = mq->parent; 2681 struct oce_cq *cq = mq->cq; 2682 int num_cqes = 0, evt_type = 0, optype = 0; 2683 struct oce_mq_cqe *cqe; 2684 struct oce_async_cqe_link_state *acqe; 2685 struct oce_async_event_qnq *dbgcqe; 2686 2687 2688 bus_dmamap_sync(cq->ring->dma.tag, 2689 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2690 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe); 2691 2692 while (cqe->u0.dw[3]) { 2693 DW_SWAP((uint32_t *) cqe, sizeof(oce_mq_cqe)); 2694 if (cqe->u0.s.async_event) { 2695 evt_type = cqe->u0.s.event_type; 2696 optype = cqe->u0.s.async_type; 2697 if (evt_type == ASYNC_EVENT_CODE_LINK_STATE) { 2698 /* Link status evt */ 2699 acqe = (struct oce_async_cqe_link_state *)cqe; 2700 process_link_state(sc, acqe); 2701 } else if (evt_type == ASYNC_EVENT_GRP5) { 2702 oce_process_grp5_events(sc, cqe); 2703 } else if (evt_type == ASYNC_EVENT_CODE_DEBUG && 2704 optype == ASYNC_EVENT_DEBUG_QNQ) { 2705 dbgcqe = (struct oce_async_event_qnq *)cqe; 2706 if(dbgcqe->valid) 2707 sc->qnqid = dbgcqe->vlan_tag; 2708 sc->qnq_debug_event = TRUE; 2709 } 2710 } 2711 cqe->u0.dw[3] = 0; 2712 RING_GET(cq->ring, 1); 2713 bus_dmamap_sync(cq->ring->dma.tag, 2714 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2715 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe); 2716 num_cqes++; 2717 } 2718 2719 if (num_cqes) 2720 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); 2721 2722 return 0; 2723 } 2724 2725 2726 static void 2727 setup_max_queues_want(POCE_SOFTC sc) 2728 { 2729 /* Check if it is FLEX machine. Is so dont use RSS */ 2730 if ((sc->function_mode & FNM_FLEX10_MODE) || 2731 (sc->function_mode & FNM_UMC_MODE) || 2732 (sc->function_mode & FNM_VNIC_MODE) || 2733 (!is_rss_enabled(sc)) || 2734 IS_BE2(sc)) { 2735 sc->nrqs = 1; 2736 sc->nwqs = 1; 2737 } else { 2738 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1; 2739 sc->nwqs = MIN(OCE_NCPUS, sc->nrssqs); 2740 } 2741 2742 if (IS_BE2(sc) && is_rss_enabled(sc)) 2743 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1; 2744 } 2745 2746 2747 static void 2748 update_queues_got(POCE_SOFTC sc) 2749 { 2750 if (is_rss_enabled(sc)) { 2751 sc->nrqs = sc->intr_count + 1; 2752 sc->nwqs = sc->intr_count; 2753 } else { 2754 sc->nrqs = 1; 2755 sc->nwqs = 1; 2756 } 2757 2758 if (IS_BE2(sc)) 2759 sc->nwqs = 1; 2760 } 2761 2762 static int 2763 oce_check_ipv6_ext_hdr(struct mbuf *m) 2764 { 2765 struct ether_header *eh = mtod(m, struct ether_header *); 2766 caddr_t m_datatemp = m->m_data; 2767 2768 if (eh->ether_type == htons(ETHERTYPE_IPV6)) { 2769 m->m_data += sizeof(struct ether_header); 2770 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); 2771 2772 if((ip6->ip6_nxt != IPPROTO_TCP) && \ 2773 (ip6->ip6_nxt != IPPROTO_UDP)){ 2774 struct ip6_ext *ip6e = NULL; 2775 m->m_data += sizeof(struct ip6_hdr); 2776 2777 ip6e = (struct ip6_ext *) mtod(m, struct ip6_ext *); 2778 if(ip6e->ip6e_len == 0xff) { 2779 m->m_data = m_datatemp; 2780 return TRUE; 2781 } 2782 } 2783 m->m_data = m_datatemp; 2784 } 2785 return FALSE; 2786 } 2787 2788 static int 2789 is_be3_a1(POCE_SOFTC sc) 2790 { 2791 if((sc->flags & OCE_FLAGS_BE3) && ((sc->asic_revision & 0xFF) < 2)) { 2792 return TRUE; 2793 } 2794 return FALSE; 2795 } 2796 2797 static struct mbuf * 2798 oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete) 2799 { 2800 uint16_t vlan_tag = 0; 2801 2802 if(!M_WRITABLE(m)) 2803 return NULL; 2804 2805 /* Embed vlan tag in the packet if it is not part of it */ 2806 if(m->m_flags & M_VLANTAG) { 2807 vlan_tag = EVL_VLANOFTAG(m->m_pkthdr.ether_vtag); 2808 m->m_flags &= ~M_VLANTAG; 2809 } 2810 2811 /* if UMC, ignore vlan tag insertion and instead insert pvid */ 2812 if(sc->pvid) { 2813 if(!vlan_tag) 2814 vlan_tag = sc->pvid; 2815 if (complete) 2816 *complete = FALSE; 2817 } 2818 2819 if(vlan_tag) { 2820 m = ether_vlanencap(m, vlan_tag); 2821 } 2822 2823 if(sc->qnqid) { 2824 m = ether_vlanencap(m, sc->qnqid); 2825 2826 if (complete) 2827 *complete = FALSE; 2828 } 2829 return m; 2830 } 2831 2832 static int 2833 oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m) 2834 { 2835 if(is_be3_a1(sc) && IS_QNQ_OR_UMC(sc) && \ 2836 oce_check_ipv6_ext_hdr(m)) { 2837 return TRUE; 2838 } 2839 return FALSE; 2840 } 2841 2842 static void 2843 oce_get_config(POCE_SOFTC sc) 2844 { 2845 int rc = 0; 2846 uint32_t max_rss = 0; 2847 2848 if ((IS_BE(sc) || IS_SH(sc)) && (!sc->be3_native)) 2849 max_rss = OCE_LEGACY_MODE_RSS; 2850 else 2851 max_rss = OCE_MAX_RSS; 2852 2853 if (!IS_BE(sc)) { 2854 rc = oce_get_profile_config(sc, max_rss); 2855 if (rc) { 2856 sc->nwqs = OCE_MAX_WQ; 2857 sc->nrssqs = max_rss; 2858 sc->nrqs = sc->nrssqs + 1; 2859 } 2860 } 2861 else { /* For BE3 don't rely on fw for determining the resources */ 2862 sc->nrssqs = max_rss; 2863 sc->nrqs = sc->nrssqs + 1; 2864 sc->nwqs = OCE_MAX_WQ; 2865 sc->max_vlans = MAX_VLANFILTER_SIZE; 2866 } 2867 } 2868 2869 static void 2870 oce_rdma_close(void) 2871 { 2872 if (oce_rdma_if != NULL) { 2873 oce_rdma_if = NULL; 2874 } 2875 } 2876 2877 static void 2878 oce_get_mac_addr(POCE_SOFTC sc, uint8_t *macaddr) 2879 { 2880 memcpy(macaddr, sc->macaddr.mac_addr, 6); 2881 } 2882 2883 int 2884 oce_register_rdma(POCE_RDMA_INFO rdma_info, POCE_RDMA_IF rdma_if) 2885 { 2886 POCE_SOFTC sc; 2887 struct oce_dev_info di; 2888 int i; 2889 2890 if ((rdma_info == NULL) || (rdma_if == NULL)) { 2891 return -EINVAL; 2892 } 2893 2894 if ((rdma_info->size != OCE_RDMA_INFO_SIZE) || 2895 (rdma_if->size != OCE_RDMA_IF_SIZE)) { 2896 return -ENXIO; 2897 } 2898 2899 rdma_info->close = oce_rdma_close; 2900 rdma_info->mbox_post = oce_mbox_post; 2901 rdma_info->common_req_hdr_init = mbx_common_req_hdr_init; 2902 rdma_info->get_mac_addr = oce_get_mac_addr; 2903 2904 oce_rdma_if = rdma_if; 2905 2906 sc = softc_head; 2907 while (sc != NULL) { 2908 if (oce_rdma_if->announce != NULL) { 2909 memset(&di, 0, sizeof(di)); 2910 di.dev = sc->dev; 2911 di.softc = sc; 2912 di.ifp = sc->ifp; 2913 di.db_bhandle = sc->db_bhandle; 2914 di.db_btag = sc->db_btag; 2915 di.db_page_size = 4096; 2916 if (sc->flags & OCE_FLAGS_USING_MSIX) { 2917 di.intr_mode = OCE_INTERRUPT_MODE_MSIX; 2918 } else if (sc->flags & OCE_FLAGS_USING_MSI) { 2919 di.intr_mode = OCE_INTERRUPT_MODE_MSI; 2920 } else { 2921 di.intr_mode = OCE_INTERRUPT_MODE_INTX; 2922 } 2923 di.dev_family = OCE_GEN2_FAMILY; // fixme: must detect skyhawk 2924 if (di.intr_mode != OCE_INTERRUPT_MODE_INTX) { 2925 di.msix.num_vectors = sc->intr_count + sc->roce_intr_count; 2926 di.msix.start_vector = sc->intr_count; 2927 for (i=0; i<di.msix.num_vectors; i++) { 2928 di.msix.vector_list[i] = sc->intrs[i].vector; 2929 } 2930 } else { 2931 } 2932 memcpy(di.mac_addr, sc->macaddr.mac_addr, 6); 2933 di.vendor_id = pci_get_vendor(sc->dev); 2934 di.dev_id = pci_get_device(sc->dev); 2935 2936 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 2937 di.flags |= OCE_RDMA_INFO_RDMA_SUPPORTED; 2938 } 2939 2940 rdma_if->announce(&di); 2941 sc = sc->next; 2942 } 2943 } 2944 2945 return 0; 2946 } 2947 2948 static void 2949 oce_read_env_variables( POCE_SOFTC sc ) 2950 { 2951 char *value = NULL; 2952 int rc = 0; 2953 2954 /* read if user wants to enable hwlro or swlro */ 2955 //value = getenv("oce_enable_hwlro"); 2956 if(value && IS_SH(sc)) { 2957 sc->enable_hwlro = strtol(value, NULL, 10); 2958 if(sc->enable_hwlro) { 2959 rc = oce_mbox_nic_query_lro_capabilities(sc, NULL, NULL); 2960 if(rc) { 2961 device_printf(sc->dev, "no hardware lro support\n"); 2962 device_printf(sc->dev, "software lro enabled\n"); 2963 sc->enable_hwlro = 0; 2964 }else { 2965 device_printf(sc->dev, "hardware lro enabled\n"); 2966 oce_max_rsp_handled = 32; 2967 } 2968 }else { 2969 device_printf(sc->dev, "software lro enabled\n"); 2970 } 2971 }else { 2972 sc->enable_hwlro = 0; 2973 } 2974 2975 /* read mbuf size */ 2976 //value = getenv("oce_rq_buf_size"); 2977 if(value && IS_SH(sc)) { 2978 oce_rq_buf_size = strtol(value, NULL, 10); 2979 switch(oce_rq_buf_size) { 2980 case 2048: 2981 case 4096: 2982 case 9216: 2983 case 16384: 2984 break; 2985 2986 default: 2987 device_printf(sc->dev, " Supported oce_rq_buf_size values are 2K, 4K, 9K, 16K \n"); 2988 oce_rq_buf_size = 2048; 2989 } 2990 } 2991 2992 return; 2993 } 2994