1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (C) 2013 Emulex 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Emulex Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 * 33 * Contact Information: 34 * freebsd-drivers@emulex.com 35 * 36 * Emulex 37 * 3333 Susan Street 38 * Costa Mesa, CA 92626 39 */ 40 41 /* $FreeBSD$ */ 42 43 #include "opt_inet6.h" 44 #include "opt_inet.h" 45 46 #include "oce_if.h" 47 #include "oce_user.h" 48 49 #define is_tso_pkt(m) (m->m_pkthdr.csum_flags & CSUM_TSO) 50 51 /* UE Status Low CSR */ 52 static char *ue_status_low_desc[] = { 53 "CEV", 54 "CTX", 55 "DBUF", 56 "ERX", 57 "Host", 58 "MPU", 59 "NDMA", 60 "PTC ", 61 "RDMA ", 62 "RXF ", 63 "RXIPS ", 64 "RXULP0 ", 65 "RXULP1 ", 66 "RXULP2 ", 67 "TIM ", 68 "TPOST ", 69 "TPRE ", 70 "TXIPS ", 71 "TXULP0 ", 72 "TXULP1 ", 73 "UC ", 74 "WDMA ", 75 "TXULP2 ", 76 "HOST1 ", 77 "P0_OB_LINK ", 78 "P1_OB_LINK ", 79 "HOST_GPIO ", 80 "MBOX ", 81 "AXGMAC0", 82 "AXGMAC1", 83 "JTAG", 84 "MPU_INTPEND" 85 }; 86 87 /* UE Status High CSR */ 88 static char *ue_status_hi_desc[] = { 89 "LPCMEMHOST", 90 "MGMT_MAC", 91 "PCS0ONLINE", 92 "MPU_IRAM", 93 "PCS1ONLINE", 94 "PCTL0", 95 "PCTL1", 96 "PMEM", 97 "RR", 98 "TXPB", 99 "RXPP", 100 "XAUI", 101 "TXP", 102 "ARM", 103 "IPC", 104 "HOST2", 105 "HOST3", 106 "HOST4", 107 "HOST5", 108 "HOST6", 109 "HOST7", 110 "HOST8", 111 "HOST9", 112 "NETC", 113 "Unknown", 114 "Unknown", 115 "Unknown", 116 "Unknown", 117 "Unknown", 118 "Unknown", 119 "Unknown", 120 "Unknown" 121 }; 122 123 struct oce_common_cqe_info{ 124 uint8_t vtp:1; 125 uint8_t l4_cksum_pass:1; 126 uint8_t ip_cksum_pass:1; 127 uint8_t ipv6_frame:1; 128 uint8_t qnq:1; 129 uint8_t rsvd:3; 130 uint8_t num_frags; 131 uint16_t pkt_size; 132 uint16_t vtag; 133 }; 134 135 136 /* Driver entry points prototypes */ 137 static int oce_probe(device_t dev); 138 static int oce_attach(device_t dev); 139 static int oce_detach(device_t dev); 140 static int oce_shutdown(device_t dev); 141 static int oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data); 142 static void oce_init(void *xsc); 143 static int oce_multiq_start(struct ifnet *ifp, struct mbuf *m); 144 static void oce_multiq_flush(struct ifnet *ifp); 145 146 /* Driver interrupt routines protypes */ 147 static void oce_intr(void *arg, int pending); 148 static int oce_setup_intr(POCE_SOFTC sc); 149 static int oce_fast_isr(void *arg); 150 static int oce_alloc_intr(POCE_SOFTC sc, int vector, 151 void (*isr) (void *arg, int pending)); 152 153 /* Media callbacks prototypes */ 154 static void oce_media_status(struct ifnet *ifp, struct ifmediareq *req); 155 static int oce_media_change(struct ifnet *ifp); 156 157 /* Transmit routines prototypes */ 158 static int oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index); 159 static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq); 160 static void oce_process_tx_completion(struct oce_wq *wq); 161 static int oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, 162 struct oce_wq *wq); 163 164 /* Receive routines prototypes */ 165 static int oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe); 166 static int oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe); 167 static void oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe); 168 static void oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq); 169 static uint16_t oce_rq_handler_lro(void *arg); 170 static void oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2); 171 static void oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2); 172 static void oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m); 173 174 /* Helper function prototypes in this file */ 175 static int oce_attach_ifp(POCE_SOFTC sc); 176 static void oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag); 177 static void oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag); 178 static int oce_vid_config(POCE_SOFTC sc); 179 static void oce_mac_addr_set(POCE_SOFTC sc); 180 static int oce_handle_passthrough(struct ifnet *ifp, caddr_t data); 181 static void oce_local_timer(void *arg); 182 static void oce_if_deactivate(POCE_SOFTC sc); 183 static void oce_if_activate(POCE_SOFTC sc); 184 static void setup_max_queues_want(POCE_SOFTC sc); 185 static void update_queues_got(POCE_SOFTC sc); 186 static void process_link_state(POCE_SOFTC sc, 187 struct oce_async_cqe_link_state *acqe); 188 static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m); 189 static void oce_get_config(POCE_SOFTC sc); 190 static struct mbuf *oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete); 191 static void oce_read_env_variables(POCE_SOFTC sc); 192 193 194 /* IP specific */ 195 #if defined(INET6) || defined(INET) 196 static int oce_init_lro(POCE_SOFTC sc); 197 static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp); 198 #endif 199 200 static device_method_t oce_dispatch[] = { 201 DEVMETHOD(device_probe, oce_probe), 202 DEVMETHOD(device_attach, oce_attach), 203 DEVMETHOD(device_detach, oce_detach), 204 DEVMETHOD(device_shutdown, oce_shutdown), 205 206 DEVMETHOD_END 207 }; 208 209 static driver_t oce_driver = { 210 "oce", 211 oce_dispatch, 212 sizeof(OCE_SOFTC) 213 }; 214 static devclass_t oce_devclass; 215 216 217 /* global vars */ 218 const char component_revision[32] = {"///" COMPONENT_REVISION "///"}; 219 220 /* Module capabilites and parameters */ 221 uint32_t oce_max_rsp_handled = OCE_MAX_RSP_HANDLED; 222 uint32_t oce_enable_rss = OCE_MODCAP_RSS; 223 uint32_t oce_rq_buf_size = 2048; 224 225 TUNABLE_INT("hw.oce.max_rsp_handled", &oce_max_rsp_handled); 226 TUNABLE_INT("hw.oce.enable_rss", &oce_enable_rss); 227 228 229 /* Supported devices table */ 230 static uint32_t supportedDevices[] = { 231 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE2, 232 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE3, 233 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_BE3, 234 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201, 235 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201_VF, 236 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_SH 237 }; 238 239 240 DRIVER_MODULE(oce, pci, oce_driver, oce_devclass, 0, 0); 241 MODULE_PNP_INFO("W32:vendor/device", pci, oce, supportedDevices, 242 nitems(supportedDevices)); 243 MODULE_DEPEND(oce, pci, 1, 1, 1); 244 MODULE_DEPEND(oce, ether, 1, 1, 1); 245 MODULE_VERSION(oce, 1); 246 247 248 POCE_SOFTC softc_head = NULL; 249 POCE_SOFTC softc_tail = NULL; 250 251 struct oce_rdma_if *oce_rdma_if = NULL; 252 253 /***************************************************************************** 254 * Driver entry points functions * 255 *****************************************************************************/ 256 257 static int 258 oce_probe(device_t dev) 259 { 260 uint16_t vendor = 0; 261 uint16_t device = 0; 262 int i = 0; 263 char str[256] = {0}; 264 POCE_SOFTC sc; 265 266 sc = device_get_softc(dev); 267 bzero(sc, sizeof(OCE_SOFTC)); 268 sc->dev = dev; 269 270 vendor = pci_get_vendor(dev); 271 device = pci_get_device(dev); 272 273 for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint32_t)); i++) { 274 if (vendor == ((supportedDevices[i] >> 16) & 0xffff)) { 275 if (device == (supportedDevices[i] & 0xffff)) { 276 sprintf(str, "%s:%s", "Emulex CNA NIC function", 277 component_revision); 278 device_set_desc_copy(dev, str); 279 280 switch (device) { 281 case PCI_PRODUCT_BE2: 282 sc->flags |= OCE_FLAGS_BE2; 283 break; 284 case PCI_PRODUCT_BE3: 285 sc->flags |= OCE_FLAGS_BE3; 286 break; 287 case PCI_PRODUCT_XE201: 288 case PCI_PRODUCT_XE201_VF: 289 sc->flags |= OCE_FLAGS_XE201; 290 break; 291 case PCI_PRODUCT_SH: 292 sc->flags |= OCE_FLAGS_SH; 293 break; 294 default: 295 return ENXIO; 296 } 297 return BUS_PROBE_DEFAULT; 298 } 299 } 300 } 301 302 return ENXIO; 303 } 304 305 306 static int 307 oce_attach(device_t dev) 308 { 309 POCE_SOFTC sc; 310 int rc = 0; 311 312 sc = device_get_softc(dev); 313 314 rc = oce_hw_pci_alloc(sc); 315 if (rc) 316 return rc; 317 318 sc->tx_ring_size = OCE_TX_RING_SIZE; 319 sc->rx_ring_size = OCE_RX_RING_SIZE; 320 /* receive fragment size should be multiple of 2K */ 321 sc->rq_frag_size = ((oce_rq_buf_size / 2048) * 2048); 322 sc->flow_control = OCE_DEFAULT_FLOW_CONTROL; 323 sc->promisc = OCE_DEFAULT_PROMISCUOUS; 324 325 LOCK_CREATE(&sc->bmbx_lock, "Mailbox_lock"); 326 LOCK_CREATE(&sc->dev_lock, "Device_lock"); 327 328 /* initialise the hardware */ 329 rc = oce_hw_init(sc); 330 if (rc) 331 goto pci_res_free; 332 333 oce_read_env_variables(sc); 334 335 oce_get_config(sc); 336 337 setup_max_queues_want(sc); 338 339 rc = oce_setup_intr(sc); 340 if (rc) 341 goto mbox_free; 342 343 rc = oce_queue_init_all(sc); 344 if (rc) 345 goto intr_free; 346 347 rc = oce_attach_ifp(sc); 348 if (rc) 349 goto queues_free; 350 351 #if defined(INET6) || defined(INET) 352 rc = oce_init_lro(sc); 353 if (rc) 354 goto ifp_free; 355 #endif 356 357 rc = oce_hw_start(sc); 358 if (rc) 359 goto lro_free; 360 361 sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 362 oce_add_vlan, sc, EVENTHANDLER_PRI_FIRST); 363 sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 364 oce_del_vlan, sc, EVENTHANDLER_PRI_FIRST); 365 366 rc = oce_stats_init(sc); 367 if (rc) 368 goto vlan_free; 369 370 oce_add_sysctls(sc); 371 372 callout_init(&sc->timer, CALLOUT_MPSAFE); 373 rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc); 374 if (rc) 375 goto stats_free; 376 377 sc->next =NULL; 378 if (softc_tail != NULL) { 379 softc_tail->next = sc; 380 } else { 381 softc_head = sc; 382 } 383 softc_tail = sc; 384 385 return 0; 386 387 stats_free: 388 callout_drain(&sc->timer); 389 oce_stats_free(sc); 390 vlan_free: 391 if (sc->vlan_attach) 392 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); 393 if (sc->vlan_detach) 394 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); 395 oce_hw_intr_disable(sc); 396 lro_free: 397 #if defined(INET6) || defined(INET) 398 oce_free_lro(sc); 399 ifp_free: 400 #endif 401 ether_ifdetach(sc->ifp); 402 if_free(sc->ifp); 403 queues_free: 404 oce_queue_release_all(sc); 405 intr_free: 406 oce_intr_free(sc); 407 mbox_free: 408 oce_dma_free(sc, &sc->bsmbx); 409 pci_res_free: 410 oce_hw_pci_free(sc); 411 LOCK_DESTROY(&sc->dev_lock); 412 LOCK_DESTROY(&sc->bmbx_lock); 413 return rc; 414 415 } 416 417 418 static int 419 oce_detach(device_t dev) 420 { 421 POCE_SOFTC sc = device_get_softc(dev); 422 POCE_SOFTC poce_sc_tmp, *ppoce_sc_tmp1, poce_sc_tmp2 = NULL; 423 424 poce_sc_tmp = softc_head; 425 ppoce_sc_tmp1 = &softc_head; 426 while (poce_sc_tmp != NULL) { 427 if (poce_sc_tmp == sc) { 428 *ppoce_sc_tmp1 = sc->next; 429 if (sc->next == NULL) { 430 softc_tail = poce_sc_tmp2; 431 } 432 break; 433 } 434 poce_sc_tmp2 = poce_sc_tmp; 435 ppoce_sc_tmp1 = &poce_sc_tmp->next; 436 poce_sc_tmp = poce_sc_tmp->next; 437 } 438 439 LOCK(&sc->dev_lock); 440 oce_if_deactivate(sc); 441 UNLOCK(&sc->dev_lock); 442 443 callout_drain(&sc->timer); 444 445 if (sc->vlan_attach != NULL) 446 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); 447 if (sc->vlan_detach != NULL) 448 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); 449 450 ether_ifdetach(sc->ifp); 451 452 if_free(sc->ifp); 453 454 oce_hw_shutdown(sc); 455 456 bus_generic_detach(dev); 457 458 return 0; 459 } 460 461 462 static int 463 oce_shutdown(device_t dev) 464 { 465 int rc; 466 467 rc = oce_detach(dev); 468 469 return rc; 470 } 471 472 473 static int 474 oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 475 { 476 struct ifreq *ifr = (struct ifreq *)data; 477 POCE_SOFTC sc = ifp->if_softc; 478 struct ifi2creq i2c; 479 uint8_t offset = 0; 480 int rc = 0; 481 uint32_t u; 482 483 switch (command) { 484 485 case SIOCGIFMEDIA: 486 rc = ifmedia_ioctl(ifp, ifr, &sc->media, command); 487 break; 488 489 case SIOCSIFMTU: 490 if (ifr->ifr_mtu > OCE_MAX_MTU) 491 rc = EINVAL; 492 else 493 ifp->if_mtu = ifr->ifr_mtu; 494 break; 495 496 case SIOCSIFFLAGS: 497 if (ifp->if_flags & IFF_UP) { 498 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 499 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; 500 oce_init(sc); 501 } 502 device_printf(sc->dev, "Interface Up\n"); 503 } else { 504 LOCK(&sc->dev_lock); 505 506 sc->ifp->if_drv_flags &= 507 ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 508 oce_if_deactivate(sc); 509 510 UNLOCK(&sc->dev_lock); 511 512 device_printf(sc->dev, "Interface Down\n"); 513 } 514 515 if ((ifp->if_flags & IFF_PROMISC) && !sc->promisc) { 516 if (!oce_rxf_set_promiscuous(sc, (1 | (1 << 1)))) 517 sc->promisc = TRUE; 518 } else if (!(ifp->if_flags & IFF_PROMISC) && sc->promisc) { 519 if (!oce_rxf_set_promiscuous(sc, 0)) 520 sc->promisc = FALSE; 521 } 522 523 break; 524 525 case SIOCADDMULTI: 526 case SIOCDELMULTI: 527 rc = oce_hw_update_multicast(sc); 528 if (rc) 529 device_printf(sc->dev, 530 "Update multicast address failed\n"); 531 break; 532 533 case SIOCSIFCAP: 534 u = ifr->ifr_reqcap ^ ifp->if_capenable; 535 536 if (u & IFCAP_TXCSUM) { 537 ifp->if_capenable ^= IFCAP_TXCSUM; 538 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 539 540 if (IFCAP_TSO & ifp->if_capenable && 541 !(IFCAP_TXCSUM & ifp->if_capenable)) { 542 u &= ~IFCAP_TSO; 543 ifp->if_capenable &= ~IFCAP_TSO; 544 ifp->if_hwassist &= ~CSUM_TSO; 545 if_printf(ifp, 546 "TSO disabled due to -txcsum.\n"); 547 } 548 } 549 550 if (u & IFCAP_RXCSUM) 551 ifp->if_capenable ^= IFCAP_RXCSUM; 552 553 if (u & IFCAP_TSO4) { 554 ifp->if_capenable ^= IFCAP_TSO4; 555 556 if (IFCAP_TSO & ifp->if_capenable) { 557 if (IFCAP_TXCSUM & ifp->if_capenable) 558 ifp->if_hwassist |= CSUM_TSO; 559 else { 560 ifp->if_capenable &= ~IFCAP_TSO; 561 ifp->if_hwassist &= ~CSUM_TSO; 562 if_printf(ifp, 563 "Enable txcsum first.\n"); 564 rc = EAGAIN; 565 } 566 } else 567 ifp->if_hwassist &= ~CSUM_TSO; 568 } 569 570 if (u & IFCAP_VLAN_HWTAGGING) 571 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 572 573 if (u & IFCAP_VLAN_HWFILTER) { 574 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 575 oce_vid_config(sc); 576 } 577 #if defined(INET6) || defined(INET) 578 if (u & IFCAP_LRO) { 579 ifp->if_capenable ^= IFCAP_LRO; 580 if(sc->enable_hwlro) { 581 if(ifp->if_capenable & IFCAP_LRO) { 582 rc = oce_mbox_nic_set_iface_lro_config(sc, 1); 583 }else { 584 rc = oce_mbox_nic_set_iface_lro_config(sc, 0); 585 } 586 } 587 } 588 #endif 589 590 break; 591 592 case SIOCGI2C: 593 rc = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c)); 594 if (rc) 595 break; 596 597 if (i2c.dev_addr == PAGE_NUM_A0) { 598 offset = i2c.offset; 599 } else if (i2c.dev_addr == PAGE_NUM_A2) { 600 offset = TRANSCEIVER_A0_SIZE + i2c.offset; 601 } else { 602 rc = EINVAL; 603 break; 604 } 605 606 if (i2c.len > sizeof(i2c.data) || 607 i2c.len + offset > sizeof(sfp_vpd_dump_buffer)) { 608 rc = EINVAL; 609 break; 610 } 611 612 rc = oce_mbox_read_transrecv_data(sc, i2c.dev_addr); 613 if (rc) { 614 rc = -rc; 615 break; 616 } 617 618 memcpy(&i2c.data[0], &sfp_vpd_dump_buffer[offset], i2c.len); 619 620 rc = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c)); 621 break; 622 623 case SIOCGPRIVATE_0: 624 rc = priv_check(curthread, PRIV_DRIVER); 625 if (rc != 0) 626 break; 627 rc = oce_handle_passthrough(ifp, data); 628 break; 629 default: 630 rc = ether_ioctl(ifp, command, data); 631 break; 632 } 633 634 return rc; 635 } 636 637 638 static void 639 oce_init(void *arg) 640 { 641 POCE_SOFTC sc = arg; 642 643 LOCK(&sc->dev_lock); 644 645 if (sc->ifp->if_flags & IFF_UP) { 646 oce_if_deactivate(sc); 647 oce_if_activate(sc); 648 } 649 650 UNLOCK(&sc->dev_lock); 651 652 } 653 654 655 static int 656 oce_multiq_start(struct ifnet *ifp, struct mbuf *m) 657 { 658 POCE_SOFTC sc = ifp->if_softc; 659 struct oce_wq *wq = NULL; 660 int queue_index = 0; 661 int status = 0; 662 663 if (!sc->link_status) 664 return ENXIO; 665 666 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 667 queue_index = m->m_pkthdr.flowid % sc->nwqs; 668 669 wq = sc->wq[queue_index]; 670 671 LOCK(&wq->tx_lock); 672 status = oce_multiq_transmit(ifp, m, wq); 673 UNLOCK(&wq->tx_lock); 674 675 return status; 676 677 } 678 679 680 static void 681 oce_multiq_flush(struct ifnet *ifp) 682 { 683 POCE_SOFTC sc = ifp->if_softc; 684 struct mbuf *m; 685 int i = 0; 686 687 for (i = 0; i < sc->nwqs; i++) { 688 while ((m = buf_ring_dequeue_sc(sc->wq[i]->br)) != NULL) 689 m_freem(m); 690 } 691 if_qflush(ifp); 692 } 693 694 695 696 /***************************************************************************** 697 * Driver interrupt routines functions * 698 *****************************************************************************/ 699 700 static void 701 oce_intr(void *arg, int pending) 702 { 703 704 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg; 705 POCE_SOFTC sc = ii->sc; 706 struct oce_eq *eq = ii->eq; 707 struct oce_eqe *eqe; 708 struct oce_cq *cq = NULL; 709 int i, num_eqes = 0; 710 711 712 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, 713 BUS_DMASYNC_POSTWRITE); 714 do { 715 eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe); 716 if (eqe->evnt == 0) 717 break; 718 eqe->evnt = 0; 719 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, 720 BUS_DMASYNC_POSTWRITE); 721 RING_GET(eq->ring, 1); 722 num_eqes++; 723 724 } while (TRUE); 725 726 if (!num_eqes) 727 goto eq_arm; /* Spurious */ 728 729 /* Clear EQ entries, but dont arm */ 730 oce_arm_eq(sc, eq->eq_id, num_eqes, FALSE, FALSE); 731 732 /* Process TX, RX and MCC. But dont arm CQ*/ 733 for (i = 0; i < eq->cq_valid; i++) { 734 cq = eq->cq[i]; 735 (*cq->cq_handler)(cq->cb_arg); 736 } 737 738 /* Arm all cqs connected to this EQ */ 739 for (i = 0; i < eq->cq_valid; i++) { 740 cq = eq->cq[i]; 741 oce_arm_cq(sc, cq->cq_id, 0, TRUE); 742 } 743 744 eq_arm: 745 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); 746 747 return; 748 } 749 750 751 static int 752 oce_setup_intr(POCE_SOFTC sc) 753 { 754 int rc = 0, use_intx = 0; 755 int vector = 0, req_vectors = 0; 756 int tot_req_vectors, tot_vectors; 757 758 if (is_rss_enabled(sc)) 759 req_vectors = MAX((sc->nrqs - 1), sc->nwqs); 760 else 761 req_vectors = 1; 762 763 tot_req_vectors = req_vectors; 764 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 765 if (req_vectors > 1) { 766 tot_req_vectors += OCE_RDMA_VECTORS; 767 sc->roce_intr_count = OCE_RDMA_VECTORS; 768 } 769 } 770 771 if (sc->flags & OCE_FLAGS_MSIX_CAPABLE) { 772 sc->intr_count = req_vectors; 773 tot_vectors = tot_req_vectors; 774 rc = pci_alloc_msix(sc->dev, &tot_vectors); 775 if (rc != 0) { 776 use_intx = 1; 777 pci_release_msi(sc->dev); 778 } else { 779 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 780 if (tot_vectors < tot_req_vectors) { 781 if (sc->intr_count < (2 * OCE_RDMA_VECTORS)) { 782 sc->roce_intr_count = (tot_vectors / 2); 783 } 784 sc->intr_count = tot_vectors - sc->roce_intr_count; 785 } 786 } else { 787 sc->intr_count = tot_vectors; 788 } 789 sc->flags |= OCE_FLAGS_USING_MSIX; 790 } 791 } else 792 use_intx = 1; 793 794 if (use_intx) 795 sc->intr_count = 1; 796 797 /* Scale number of queues based on intr we got */ 798 update_queues_got(sc); 799 800 if (use_intx) { 801 device_printf(sc->dev, "Using legacy interrupt\n"); 802 rc = oce_alloc_intr(sc, vector, oce_intr); 803 if (rc) 804 goto error; 805 } else { 806 for (; vector < sc->intr_count; vector++) { 807 rc = oce_alloc_intr(sc, vector, oce_intr); 808 if (rc) 809 goto error; 810 } 811 } 812 813 return 0; 814 error: 815 oce_intr_free(sc); 816 return rc; 817 } 818 819 820 static int 821 oce_fast_isr(void *arg) 822 { 823 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg; 824 POCE_SOFTC sc = ii->sc; 825 826 if (ii->eq == NULL) 827 return FILTER_STRAY; 828 829 oce_arm_eq(sc, ii->eq->eq_id, 0, FALSE, TRUE); 830 831 taskqueue_enqueue(ii->tq, &ii->task); 832 833 ii->eq->intr++; 834 835 return FILTER_HANDLED; 836 } 837 838 839 static int 840 oce_alloc_intr(POCE_SOFTC sc, int vector, void (*isr) (void *arg, int pending)) 841 { 842 POCE_INTR_INFO ii; 843 int rc = 0, rr; 844 845 if (vector >= OCE_MAX_EQ) 846 return (EINVAL); 847 848 ii = &sc->intrs[vector]; 849 850 /* Set the resource id for the interrupt. 851 * MSIx is vector + 1 for the resource id, 852 * INTx is 0 for the resource id. 853 */ 854 if (sc->flags & OCE_FLAGS_USING_MSIX) 855 rr = vector + 1; 856 else 857 rr = 0; 858 ii->intr_res = bus_alloc_resource_any(sc->dev, 859 SYS_RES_IRQ, 860 &rr, RF_ACTIVE|RF_SHAREABLE); 861 ii->irq_rr = rr; 862 if (ii->intr_res == NULL) { 863 device_printf(sc->dev, 864 "Could not allocate interrupt\n"); 865 rc = ENXIO; 866 return rc; 867 } 868 869 TASK_INIT(&ii->task, 0, isr, ii); 870 ii->vector = vector; 871 sprintf(ii->task_name, "oce_task[%d]", ii->vector); 872 ii->tq = taskqueue_create_fast(ii->task_name, 873 M_NOWAIT, 874 taskqueue_thread_enqueue, 875 &ii->tq); 876 taskqueue_start_threads(&ii->tq, 1, PI_NET, "%s taskq", 877 device_get_nameunit(sc->dev)); 878 879 ii->sc = sc; 880 rc = bus_setup_intr(sc->dev, 881 ii->intr_res, 882 INTR_TYPE_NET, 883 oce_fast_isr, NULL, ii, &ii->tag); 884 return rc; 885 886 } 887 888 889 void 890 oce_intr_free(POCE_SOFTC sc) 891 { 892 int i = 0; 893 894 for (i = 0; i < sc->intr_count; i++) { 895 896 if (sc->intrs[i].tag != NULL) 897 bus_teardown_intr(sc->dev, sc->intrs[i].intr_res, 898 sc->intrs[i].tag); 899 if (sc->intrs[i].tq != NULL) 900 taskqueue_free(sc->intrs[i].tq); 901 902 if (sc->intrs[i].intr_res != NULL) 903 bus_release_resource(sc->dev, SYS_RES_IRQ, 904 sc->intrs[i].irq_rr, 905 sc->intrs[i].intr_res); 906 sc->intrs[i].tag = NULL; 907 sc->intrs[i].intr_res = NULL; 908 } 909 910 if (sc->flags & OCE_FLAGS_USING_MSIX) 911 pci_release_msi(sc->dev); 912 913 } 914 915 916 917 /****************************************************************************** 918 * Media callbacks functions * 919 ******************************************************************************/ 920 921 static void 922 oce_media_status(struct ifnet *ifp, struct ifmediareq *req) 923 { 924 POCE_SOFTC sc = (POCE_SOFTC) ifp->if_softc; 925 926 927 req->ifm_status = IFM_AVALID; 928 req->ifm_active = IFM_ETHER; 929 930 if (sc->link_status == 1) 931 req->ifm_status |= IFM_ACTIVE; 932 else 933 return; 934 935 switch (sc->link_speed) { 936 case 1: /* 10 Mbps */ 937 req->ifm_active |= IFM_10_T | IFM_FDX; 938 sc->speed = 10; 939 break; 940 case 2: /* 100 Mbps */ 941 req->ifm_active |= IFM_100_TX | IFM_FDX; 942 sc->speed = 100; 943 break; 944 case 3: /* 1 Gbps */ 945 req->ifm_active |= IFM_1000_T | IFM_FDX; 946 sc->speed = 1000; 947 break; 948 case 4: /* 10 Gbps */ 949 req->ifm_active |= IFM_10G_SR | IFM_FDX; 950 sc->speed = 10000; 951 break; 952 case 5: /* 20 Gbps */ 953 req->ifm_active |= IFM_10G_SR | IFM_FDX; 954 sc->speed = 20000; 955 break; 956 case 6: /* 25 Gbps */ 957 req->ifm_active |= IFM_10G_SR | IFM_FDX; 958 sc->speed = 25000; 959 break; 960 case 7: /* 40 Gbps */ 961 req->ifm_active |= IFM_40G_SR4 | IFM_FDX; 962 sc->speed = 40000; 963 break; 964 default: 965 sc->speed = 0; 966 break; 967 } 968 969 return; 970 } 971 972 973 int 974 oce_media_change(struct ifnet *ifp) 975 { 976 return 0; 977 } 978 979 980 static void oce_is_pkt_dest_bmc(POCE_SOFTC sc, 981 struct mbuf *m, boolean_t *os2bmc, 982 struct mbuf **m_new) 983 { 984 struct ether_header *eh = NULL; 985 986 eh = mtod(m, struct ether_header *); 987 988 if (!is_os2bmc_enabled(sc) || *os2bmc) { 989 *os2bmc = FALSE; 990 goto done; 991 } 992 if (!ETHER_IS_MULTICAST(eh->ether_dhost)) 993 goto done; 994 995 if (is_mc_allowed_on_bmc(sc, eh) || 996 is_bc_allowed_on_bmc(sc, eh) || 997 is_arp_allowed_on_bmc(sc, ntohs(eh->ether_type))) { 998 *os2bmc = TRUE; 999 goto done; 1000 } 1001 1002 if (mtod(m, struct ip *)->ip_p == IPPROTO_IPV6) { 1003 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); 1004 uint8_t nexthdr = ip6->ip6_nxt; 1005 if (nexthdr == IPPROTO_ICMPV6) { 1006 struct icmp6_hdr *icmp6 = (struct icmp6_hdr *)(ip6 + 1); 1007 switch (icmp6->icmp6_type) { 1008 case ND_ROUTER_ADVERT: 1009 *os2bmc = is_ipv6_ra_filt_enabled(sc); 1010 goto done; 1011 case ND_NEIGHBOR_ADVERT: 1012 *os2bmc = is_ipv6_na_filt_enabled(sc); 1013 goto done; 1014 default: 1015 break; 1016 } 1017 } 1018 } 1019 1020 if (mtod(m, struct ip *)->ip_p == IPPROTO_UDP) { 1021 struct ip *ip = mtod(m, struct ip *); 1022 int iphlen = ip->ip_hl << 2; 1023 struct udphdr *uh = (struct udphdr *)((caddr_t)ip + iphlen); 1024 switch (uh->uh_dport) { 1025 case DHCP_CLIENT_PORT: 1026 *os2bmc = is_dhcp_client_filt_enabled(sc); 1027 goto done; 1028 case DHCP_SERVER_PORT: 1029 *os2bmc = is_dhcp_srvr_filt_enabled(sc); 1030 goto done; 1031 case NET_BIOS_PORT1: 1032 case NET_BIOS_PORT2: 1033 *os2bmc = is_nbios_filt_enabled(sc); 1034 goto done; 1035 case DHCPV6_RAS_PORT: 1036 *os2bmc = is_ipv6_ras_filt_enabled(sc); 1037 goto done; 1038 default: 1039 break; 1040 } 1041 } 1042 done: 1043 if (*os2bmc) { 1044 *m_new = m_dup(m, M_NOWAIT); 1045 if (!*m_new) { 1046 *os2bmc = FALSE; 1047 return; 1048 } 1049 *m_new = oce_insert_vlan_tag(sc, *m_new, NULL); 1050 } 1051 } 1052 1053 1054 1055 /***************************************************************************** 1056 * Transmit routines functions * 1057 *****************************************************************************/ 1058 1059 static int 1060 oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index) 1061 { 1062 int rc = 0, i, retry_cnt = 0; 1063 bus_dma_segment_t segs[OCE_MAX_TX_ELEMENTS]; 1064 struct mbuf *m, *m_temp, *m_new = NULL; 1065 struct oce_wq *wq = sc->wq[wq_index]; 1066 struct oce_packet_desc *pd; 1067 struct oce_nic_hdr_wqe *nichdr; 1068 struct oce_nic_frag_wqe *nicfrag; 1069 struct ether_header *eh = NULL; 1070 int num_wqes; 1071 uint32_t reg_value; 1072 boolean_t complete = TRUE; 1073 boolean_t os2bmc = FALSE; 1074 1075 m = *mpp; 1076 if (!m) 1077 return EINVAL; 1078 1079 if (!(m->m_flags & M_PKTHDR)) { 1080 rc = ENXIO; 1081 goto free_ret; 1082 } 1083 1084 /* Don't allow non-TSO packets longer than MTU */ 1085 if (!is_tso_pkt(m)) { 1086 eh = mtod(m, struct ether_header *); 1087 if(m->m_pkthdr.len > ETHER_MAX_FRAME(sc->ifp, eh->ether_type, FALSE)) 1088 goto free_ret; 1089 } 1090 1091 if(oce_tx_asic_stall_verify(sc, m)) { 1092 m = oce_insert_vlan_tag(sc, m, &complete); 1093 if(!m) { 1094 device_printf(sc->dev, "Insertion unsuccessful\n"); 1095 return 0; 1096 } 1097 1098 } 1099 1100 /* Lancer, SH ASIC has a bug wherein Packets that are 32 bytes or less 1101 * may cause a transmit stall on that port. So the work-around is to 1102 * pad short packets (<= 32 bytes) to a 36-byte length. 1103 */ 1104 if(IS_SH(sc) || IS_XE201(sc) ) { 1105 if(m->m_pkthdr.len <= 32) { 1106 char buf[36]; 1107 bzero((void *)buf, 36); 1108 m_append(m, (36 - m->m_pkthdr.len), buf); 1109 } 1110 } 1111 1112 tx_start: 1113 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1114 /* consolidate packet buffers for TSO/LSO segment offload */ 1115 #if defined(INET6) || defined(INET) 1116 m = oce_tso_setup(sc, mpp); 1117 #else 1118 m = NULL; 1119 #endif 1120 if (m == NULL) { 1121 rc = ENXIO; 1122 goto free_ret; 1123 } 1124 } 1125 1126 1127 pd = &wq->pckts[wq->pkt_desc_head]; 1128 1129 retry: 1130 rc = bus_dmamap_load_mbuf_sg(wq->tag, 1131 pd->map, 1132 m, segs, &pd->nsegs, BUS_DMA_NOWAIT); 1133 if (rc == 0) { 1134 num_wqes = pd->nsegs + 1; 1135 if (IS_BE(sc) || IS_SH(sc)) { 1136 /*Dummy required only for BE3.*/ 1137 if (num_wqes & 1) 1138 num_wqes++; 1139 } 1140 if (num_wqes >= RING_NUM_FREE(wq->ring)) { 1141 bus_dmamap_unload(wq->tag, pd->map); 1142 return EBUSY; 1143 } 1144 atomic_store_rel_int(&wq->pkt_desc_head, 1145 (wq->pkt_desc_head + 1) % \ 1146 OCE_WQ_PACKET_ARRAY_SIZE); 1147 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_PREWRITE); 1148 pd->mbuf = m; 1149 1150 nichdr = 1151 RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_hdr_wqe); 1152 nichdr->u0.dw[0] = 0; 1153 nichdr->u0.dw[1] = 0; 1154 nichdr->u0.dw[2] = 0; 1155 nichdr->u0.dw[3] = 0; 1156 1157 nichdr->u0.s.complete = complete; 1158 nichdr->u0.s.mgmt = os2bmc; 1159 nichdr->u0.s.event = 1; 1160 nichdr->u0.s.crc = 1; 1161 nichdr->u0.s.forward = 0; 1162 nichdr->u0.s.ipcs = (m->m_pkthdr.csum_flags & CSUM_IP) ? 1 : 0; 1163 nichdr->u0.s.udpcs = 1164 (m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0; 1165 nichdr->u0.s.tcpcs = 1166 (m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0; 1167 nichdr->u0.s.num_wqe = num_wqes; 1168 nichdr->u0.s.total_length = m->m_pkthdr.len; 1169 1170 if (m->m_flags & M_VLANTAG) { 1171 nichdr->u0.s.vlan = 1; /*Vlan present*/ 1172 nichdr->u0.s.vlan_tag = m->m_pkthdr.ether_vtag; 1173 } 1174 1175 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1176 if (m->m_pkthdr.tso_segsz) { 1177 nichdr->u0.s.lso = 1; 1178 nichdr->u0.s.lso_mss = m->m_pkthdr.tso_segsz; 1179 } 1180 if (!IS_BE(sc) || !IS_SH(sc)) 1181 nichdr->u0.s.ipcs = 1; 1182 } 1183 1184 RING_PUT(wq->ring, 1); 1185 atomic_add_int(&wq->ring->num_used, 1); 1186 1187 for (i = 0; i < pd->nsegs; i++) { 1188 nicfrag = 1189 RING_GET_PRODUCER_ITEM_VA(wq->ring, 1190 struct oce_nic_frag_wqe); 1191 nicfrag->u0.s.rsvd0 = 0; 1192 nicfrag->u0.s.frag_pa_hi = ADDR_HI(segs[i].ds_addr); 1193 nicfrag->u0.s.frag_pa_lo = ADDR_LO(segs[i].ds_addr); 1194 nicfrag->u0.s.frag_len = segs[i].ds_len; 1195 pd->wqe_idx = wq->ring->pidx; 1196 RING_PUT(wq->ring, 1); 1197 atomic_add_int(&wq->ring->num_used, 1); 1198 } 1199 if (num_wqes > (pd->nsegs + 1)) { 1200 nicfrag = 1201 RING_GET_PRODUCER_ITEM_VA(wq->ring, 1202 struct oce_nic_frag_wqe); 1203 nicfrag->u0.dw[0] = 0; 1204 nicfrag->u0.dw[1] = 0; 1205 nicfrag->u0.dw[2] = 0; 1206 nicfrag->u0.dw[3] = 0; 1207 pd->wqe_idx = wq->ring->pidx; 1208 RING_PUT(wq->ring, 1); 1209 atomic_add_int(&wq->ring->num_used, 1); 1210 pd->nsegs++; 1211 } 1212 1213 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1); 1214 wq->tx_stats.tx_reqs++; 1215 wq->tx_stats.tx_wrbs += num_wqes; 1216 wq->tx_stats.tx_bytes += m->m_pkthdr.len; 1217 wq->tx_stats.tx_pkts++; 1218 1219 bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map, 1220 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1221 reg_value = (num_wqes << 16) | wq->wq_id; 1222 1223 /* if os2bmc is not enabled or if the pkt is already tagged as 1224 bmc, do nothing 1225 */ 1226 oce_is_pkt_dest_bmc(sc, m, &os2bmc, &m_new); 1227 1228 if_inc_counter(sc->ifp, IFCOUNTER_OBYTES, m->m_pkthdr.len); 1229 if (m->m_flags & M_MCAST) 1230 if_inc_counter(sc->ifp, IFCOUNTER_OMCASTS, 1); 1231 ETHER_BPF_MTAP(sc->ifp, m); 1232 1233 OCE_WRITE_REG32(sc, db, wq->db_offset, reg_value); 1234 1235 } else if (rc == EFBIG) { 1236 if (retry_cnt == 0) { 1237 m_temp = m_defrag(m, M_NOWAIT); 1238 if (m_temp == NULL) 1239 goto free_ret; 1240 m = m_temp; 1241 *mpp = m_temp; 1242 retry_cnt = retry_cnt + 1; 1243 goto retry; 1244 } else 1245 goto free_ret; 1246 } else if (rc == ENOMEM) 1247 return rc; 1248 else 1249 goto free_ret; 1250 1251 if (os2bmc) { 1252 m = m_new; 1253 goto tx_start; 1254 } 1255 1256 return 0; 1257 1258 free_ret: 1259 m_freem(*mpp); 1260 *mpp = NULL; 1261 return rc; 1262 } 1263 1264 1265 static void 1266 oce_process_tx_completion(struct oce_wq *wq) 1267 { 1268 struct oce_packet_desc *pd; 1269 POCE_SOFTC sc = (POCE_SOFTC) wq->parent; 1270 struct mbuf *m; 1271 1272 pd = &wq->pckts[wq->pkt_desc_tail]; 1273 atomic_store_rel_int(&wq->pkt_desc_tail, 1274 (wq->pkt_desc_tail + 1) % OCE_WQ_PACKET_ARRAY_SIZE); 1275 atomic_subtract_int(&wq->ring->num_used, pd->nsegs + 1); 1276 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1277 bus_dmamap_unload(wq->tag, pd->map); 1278 1279 m = pd->mbuf; 1280 m_freem(m); 1281 pd->mbuf = NULL; 1282 1283 1284 if (sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) { 1285 if (wq->ring->num_used < (wq->ring->num_items / 2)) { 1286 sc->ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE); 1287 oce_tx_restart(sc, wq); 1288 } 1289 } 1290 } 1291 1292 1293 static void 1294 oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq) 1295 { 1296 1297 if ((sc->ifp->if_drv_flags & IFF_DRV_RUNNING) != IFF_DRV_RUNNING) 1298 return; 1299 1300 if (!drbr_empty(sc->ifp, wq->br)) 1301 taskqueue_enqueue(taskqueue_swi, &wq->txtask); 1302 1303 } 1304 1305 1306 #if defined(INET6) || defined(INET) 1307 static struct mbuf * 1308 oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp) 1309 { 1310 struct mbuf *m; 1311 #ifdef INET 1312 struct ip *ip; 1313 #endif 1314 #ifdef INET6 1315 struct ip6_hdr *ip6; 1316 #endif 1317 struct ether_vlan_header *eh; 1318 struct tcphdr *th; 1319 uint16_t etype; 1320 int total_len = 0, ehdrlen = 0; 1321 1322 m = *mpp; 1323 1324 if (M_WRITABLE(m) == 0) { 1325 m = m_dup(*mpp, M_NOWAIT); 1326 if (!m) 1327 return NULL; 1328 m_freem(*mpp); 1329 *mpp = m; 1330 } 1331 1332 eh = mtod(m, struct ether_vlan_header *); 1333 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 1334 etype = ntohs(eh->evl_proto); 1335 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 1336 } else { 1337 etype = ntohs(eh->evl_encap_proto); 1338 ehdrlen = ETHER_HDR_LEN; 1339 } 1340 1341 switch (etype) { 1342 #ifdef INET 1343 case ETHERTYPE_IP: 1344 ip = (struct ip *)(m->m_data + ehdrlen); 1345 if (ip->ip_p != IPPROTO_TCP) 1346 return NULL; 1347 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2)); 1348 1349 total_len = ehdrlen + (ip->ip_hl << 2) + (th->th_off << 2); 1350 break; 1351 #endif 1352 #ifdef INET6 1353 case ETHERTYPE_IPV6: 1354 ip6 = (struct ip6_hdr *)(m->m_data + ehdrlen); 1355 if (ip6->ip6_nxt != IPPROTO_TCP) 1356 return NULL; 1357 th = (struct tcphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr)); 1358 1359 total_len = ehdrlen + sizeof(struct ip6_hdr) + (th->th_off << 2); 1360 break; 1361 #endif 1362 default: 1363 return NULL; 1364 } 1365 1366 m = m_pullup(m, total_len); 1367 if (!m) 1368 return NULL; 1369 *mpp = m; 1370 return m; 1371 1372 } 1373 #endif /* INET6 || INET */ 1374 1375 void 1376 oce_tx_task(void *arg, int npending) 1377 { 1378 struct oce_wq *wq = arg; 1379 POCE_SOFTC sc = wq->parent; 1380 struct ifnet *ifp = sc->ifp; 1381 int rc = 0; 1382 1383 LOCK(&wq->tx_lock); 1384 rc = oce_multiq_transmit(ifp, NULL, wq); 1385 if (rc) { 1386 device_printf(sc->dev, 1387 "TX[%d] restart failed\n", wq->queue_index); 1388 } 1389 UNLOCK(&wq->tx_lock); 1390 } 1391 1392 1393 void 1394 oce_start(struct ifnet *ifp) 1395 { 1396 POCE_SOFTC sc = ifp->if_softc; 1397 struct mbuf *m; 1398 int rc = 0; 1399 int def_q = 0; /* Defualt tx queue is 0*/ 1400 1401 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 1402 IFF_DRV_RUNNING) 1403 return; 1404 1405 if (!sc->link_status) 1406 return; 1407 1408 while (true) { 1409 IF_DEQUEUE(&sc->ifp->if_snd, m); 1410 if (m == NULL) 1411 break; 1412 1413 LOCK(&sc->wq[def_q]->tx_lock); 1414 rc = oce_tx(sc, &m, def_q); 1415 UNLOCK(&sc->wq[def_q]->tx_lock); 1416 if (rc) { 1417 if (m != NULL) { 1418 sc->wq[def_q]->tx_stats.tx_stops ++; 1419 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1420 IFQ_DRV_PREPEND(&ifp->if_snd, m); 1421 m = NULL; 1422 } 1423 break; 1424 } 1425 } 1426 } 1427 1428 1429 /* Handle the Completion Queue for transmit */ 1430 uint16_t 1431 oce_wq_handler(void *arg) 1432 { 1433 struct oce_wq *wq = (struct oce_wq *)arg; 1434 POCE_SOFTC sc = wq->parent; 1435 struct oce_cq *cq = wq->cq; 1436 struct oce_nic_tx_cqe *cqe; 1437 int num_cqes = 0; 1438 1439 LOCK(&wq->tx_compl_lock); 1440 bus_dmamap_sync(cq->ring->dma.tag, 1441 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1442 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 1443 while (cqe->u0.dw[3]) { 1444 DW_SWAP((uint32_t *) cqe, sizeof(oce_wq_cqe)); 1445 1446 wq->ring->cidx = cqe->u0.s.wqe_index + 1; 1447 if (wq->ring->cidx >= wq->ring->num_items) 1448 wq->ring->cidx -= wq->ring->num_items; 1449 1450 oce_process_tx_completion(wq); 1451 wq->tx_stats.tx_compl++; 1452 cqe->u0.dw[3] = 0; 1453 RING_GET(cq->ring, 1); 1454 bus_dmamap_sync(cq->ring->dma.tag, 1455 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1456 cqe = 1457 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 1458 num_cqes++; 1459 } 1460 1461 if (num_cqes) 1462 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); 1463 1464 UNLOCK(&wq->tx_compl_lock); 1465 return num_cqes; 1466 } 1467 1468 1469 static int 1470 oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, struct oce_wq *wq) 1471 { 1472 POCE_SOFTC sc = ifp->if_softc; 1473 int status = 0, queue_index = 0; 1474 struct mbuf *next = NULL; 1475 struct buf_ring *br = NULL; 1476 1477 br = wq->br; 1478 queue_index = wq->queue_index; 1479 1480 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 1481 IFF_DRV_RUNNING) { 1482 if (m != NULL) 1483 status = drbr_enqueue(ifp, br, m); 1484 return status; 1485 } 1486 1487 if (m != NULL) { 1488 if ((status = drbr_enqueue(ifp, br, m)) != 0) 1489 return status; 1490 } 1491 while ((next = drbr_peek(ifp, br)) != NULL) { 1492 if (oce_tx(sc, &next, queue_index)) { 1493 if (next == NULL) { 1494 drbr_advance(ifp, br); 1495 } else { 1496 drbr_putback(ifp, br, next); 1497 wq->tx_stats.tx_stops ++; 1498 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1499 } 1500 break; 1501 } 1502 drbr_advance(ifp, br); 1503 } 1504 1505 return 0; 1506 } 1507 1508 1509 1510 1511 /***************************************************************************** 1512 * Receive routines functions * 1513 *****************************************************************************/ 1514 1515 static void 1516 oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2) 1517 { 1518 uint32_t *p; 1519 struct ether_header *eh = NULL; 1520 struct tcphdr *tcp_hdr = NULL; 1521 struct ip *ip4_hdr = NULL; 1522 struct ip6_hdr *ip6 = NULL; 1523 uint32_t payload_len = 0; 1524 1525 eh = mtod(m, struct ether_header *); 1526 /* correct IP header */ 1527 if(!cqe2->ipv6_frame) { 1528 ip4_hdr = (struct ip *)((char*)eh + sizeof(struct ether_header)); 1529 ip4_hdr->ip_ttl = cqe2->frame_lifespan; 1530 ip4_hdr->ip_len = htons(cqe2->coalesced_size - sizeof(struct ether_header)); 1531 tcp_hdr = (struct tcphdr *)((char*)ip4_hdr + sizeof(struct ip)); 1532 }else { 1533 ip6 = (struct ip6_hdr *)((char*)eh + sizeof(struct ether_header)); 1534 ip6->ip6_ctlun.ip6_un1.ip6_un1_hlim = cqe2->frame_lifespan; 1535 payload_len = cqe2->coalesced_size - sizeof(struct ether_header) 1536 - sizeof(struct ip6_hdr); 1537 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = htons(payload_len); 1538 tcp_hdr = (struct tcphdr *)((char*)ip6 + sizeof(struct ip6_hdr)); 1539 } 1540 1541 /* correct tcp header */ 1542 tcp_hdr->th_ack = htonl(cqe2->tcp_ack_num); 1543 if(cqe2->push) { 1544 tcp_hdr->th_flags |= TH_PUSH; 1545 } 1546 tcp_hdr->th_win = htons(cqe2->tcp_window); 1547 tcp_hdr->th_sum = 0xffff; 1548 if(cqe2->ts_opt) { 1549 p = (uint32_t *)((char*)tcp_hdr + sizeof(struct tcphdr) + 2); 1550 *p = cqe1->tcp_timestamp_val; 1551 *(p+1) = cqe1->tcp_timestamp_ecr; 1552 } 1553 1554 return; 1555 } 1556 1557 static void 1558 oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m) 1559 { 1560 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1561 uint32_t i = 0, frag_len = 0; 1562 uint32_t len = cqe_info->pkt_size; 1563 struct oce_packet_desc *pd; 1564 struct mbuf *tail = NULL; 1565 1566 for (i = 0; i < cqe_info->num_frags; i++) { 1567 if (rq->ring->cidx == rq->ring->pidx) { 1568 device_printf(sc->dev, 1569 "oce_rx_mbuf_chain: Invalid RX completion - Queue is empty\n"); 1570 return; 1571 } 1572 pd = &rq->pckts[rq->ring->cidx]; 1573 1574 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1575 bus_dmamap_unload(rq->tag, pd->map); 1576 RING_GET(rq->ring, 1); 1577 rq->pending--; 1578 1579 frag_len = (len > rq->cfg.frag_size) ? rq->cfg.frag_size : len; 1580 pd->mbuf->m_len = frag_len; 1581 1582 if (tail != NULL) { 1583 /* additional fragments */ 1584 pd->mbuf->m_flags &= ~M_PKTHDR; 1585 tail->m_next = pd->mbuf; 1586 if(rq->islro) 1587 tail->m_nextpkt = NULL; 1588 tail = pd->mbuf; 1589 } else { 1590 /* first fragment, fill out much of the packet header */ 1591 pd->mbuf->m_pkthdr.len = len; 1592 if(rq->islro) 1593 pd->mbuf->m_nextpkt = NULL; 1594 pd->mbuf->m_pkthdr.csum_flags = 0; 1595 if (IF_CSUM_ENABLED(sc)) { 1596 if (cqe_info->l4_cksum_pass) { 1597 if(!cqe_info->ipv6_frame) { /* IPV4 */ 1598 pd->mbuf->m_pkthdr.csum_flags |= 1599 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1600 }else { /* IPV6 frame */ 1601 if(rq->islro) { 1602 pd->mbuf->m_pkthdr.csum_flags |= 1603 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1604 } 1605 } 1606 pd->mbuf->m_pkthdr.csum_data = 0xffff; 1607 } 1608 if (cqe_info->ip_cksum_pass) { 1609 pd->mbuf->m_pkthdr.csum_flags |= 1610 (CSUM_IP_CHECKED|CSUM_IP_VALID); 1611 } 1612 } 1613 *m = tail = pd->mbuf; 1614 } 1615 pd->mbuf = NULL; 1616 len -= frag_len; 1617 } 1618 1619 return; 1620 } 1621 1622 static void 1623 oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2) 1624 { 1625 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1626 struct nic_hwlro_cqe_part1 *cqe1 = NULL; 1627 struct mbuf *m = NULL; 1628 struct oce_common_cqe_info cq_info; 1629 1630 /* parse cqe */ 1631 if(cqe2 == NULL) { 1632 cq_info.pkt_size = cqe->pkt_size; 1633 cq_info.vtag = cqe->vlan_tag; 1634 cq_info.l4_cksum_pass = cqe->l4_cksum_pass; 1635 cq_info.ip_cksum_pass = cqe->ip_cksum_pass; 1636 cq_info.ipv6_frame = cqe->ipv6_frame; 1637 cq_info.vtp = cqe->vtp; 1638 cq_info.qnq = cqe->qnq; 1639 }else { 1640 cqe1 = (struct nic_hwlro_cqe_part1 *)cqe; 1641 cq_info.pkt_size = cqe2->coalesced_size; 1642 cq_info.vtag = cqe2->vlan_tag; 1643 cq_info.l4_cksum_pass = cqe2->l4_cksum_pass; 1644 cq_info.ip_cksum_pass = cqe2->ip_cksum_pass; 1645 cq_info.ipv6_frame = cqe2->ipv6_frame; 1646 cq_info.vtp = cqe2->vtp; 1647 cq_info.qnq = cqe1->qnq; 1648 } 1649 1650 cq_info.vtag = BSWAP_16(cq_info.vtag); 1651 1652 cq_info.num_frags = cq_info.pkt_size / rq->cfg.frag_size; 1653 if(cq_info.pkt_size % rq->cfg.frag_size) 1654 cq_info.num_frags++; 1655 1656 oce_rx_mbuf_chain(rq, &cq_info, &m); 1657 1658 if (m) { 1659 if(cqe2) { 1660 //assert(cqe2->valid != 0); 1661 1662 //assert(cqe2->cqe_type != 2); 1663 oce_correct_header(m, cqe1, cqe2); 1664 } 1665 1666 m->m_pkthdr.rcvif = sc->ifp; 1667 if (rq->queue_index) 1668 m->m_pkthdr.flowid = (rq->queue_index - 1); 1669 else 1670 m->m_pkthdr.flowid = rq->queue_index; 1671 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 1672 1673 /* This deternies if vlan tag is Valid */ 1674 if (cq_info.vtp) { 1675 if (sc->function_mode & FNM_FLEX10_MODE) { 1676 /* FLEX10. If QnQ is not set, neglect VLAN */ 1677 if (cq_info.qnq) { 1678 m->m_pkthdr.ether_vtag = cq_info.vtag; 1679 m->m_flags |= M_VLANTAG; 1680 } 1681 } else if (sc->pvid != (cq_info.vtag & VLAN_VID_MASK)) { 1682 /* In UMC mode generally pvid will be striped by 1683 hw. But in some cases we have seen it comes 1684 with pvid. So if pvid == vlan, neglect vlan. 1685 */ 1686 m->m_pkthdr.ether_vtag = cq_info.vtag; 1687 m->m_flags |= M_VLANTAG; 1688 } 1689 } 1690 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 1691 1692 (*sc->ifp->if_input) (sc->ifp, m); 1693 1694 /* Update rx stats per queue */ 1695 rq->rx_stats.rx_pkts++; 1696 rq->rx_stats.rx_bytes += cq_info.pkt_size; 1697 rq->rx_stats.rx_frags += cq_info.num_frags; 1698 rq->rx_stats.rx_ucast_pkts++; 1699 } 1700 return; 1701 } 1702 1703 static void 1704 oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe) 1705 { 1706 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1707 int len; 1708 struct mbuf *m = NULL; 1709 struct oce_common_cqe_info cq_info; 1710 uint16_t vtag = 0; 1711 1712 /* Is it a flush compl that has no data */ 1713 if(!cqe->u0.s.num_fragments) 1714 goto exit; 1715 1716 len = cqe->u0.s.pkt_size; 1717 if (!len) { 1718 /*partial DMA workaround for Lancer*/ 1719 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); 1720 goto exit; 1721 } 1722 1723 if (!oce_cqe_portid_valid(sc, cqe)) { 1724 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); 1725 goto exit; 1726 } 1727 1728 /* Get vlan_tag value */ 1729 if(IS_BE(sc) || IS_SH(sc)) 1730 vtag = BSWAP_16(cqe->u0.s.vlan_tag); 1731 else 1732 vtag = cqe->u0.s.vlan_tag; 1733 1734 cq_info.l4_cksum_pass = cqe->u0.s.l4_cksum_pass; 1735 cq_info.ip_cksum_pass = cqe->u0.s.ip_cksum_pass; 1736 cq_info.ipv6_frame = cqe->u0.s.ip_ver; 1737 cq_info.num_frags = cqe->u0.s.num_fragments; 1738 cq_info.pkt_size = cqe->u0.s.pkt_size; 1739 1740 oce_rx_mbuf_chain(rq, &cq_info, &m); 1741 1742 if (m) { 1743 m->m_pkthdr.rcvif = sc->ifp; 1744 if (rq->queue_index) 1745 m->m_pkthdr.flowid = (rq->queue_index - 1); 1746 else 1747 m->m_pkthdr.flowid = rq->queue_index; 1748 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 1749 1750 /* This deternies if vlan tag is Valid */ 1751 if (oce_cqe_vtp_valid(sc, cqe)) { 1752 if (sc->function_mode & FNM_FLEX10_MODE) { 1753 /* FLEX10. If QnQ is not set, neglect VLAN */ 1754 if (cqe->u0.s.qnq) { 1755 m->m_pkthdr.ether_vtag = vtag; 1756 m->m_flags |= M_VLANTAG; 1757 } 1758 } else if (sc->pvid != (vtag & VLAN_VID_MASK)) { 1759 /* In UMC mode generally pvid will be striped by 1760 hw. But in some cases we have seen it comes 1761 with pvid. So if pvid == vlan, neglect vlan. 1762 */ 1763 m->m_pkthdr.ether_vtag = vtag; 1764 m->m_flags |= M_VLANTAG; 1765 } 1766 } 1767 1768 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 1769 #if defined(INET6) || defined(INET) 1770 /* Try to queue to LRO */ 1771 if (IF_LRO_ENABLED(sc) && 1772 (cqe->u0.s.ip_cksum_pass) && 1773 (cqe->u0.s.l4_cksum_pass) && 1774 (!cqe->u0.s.ip_ver) && 1775 (rq->lro.lro_cnt != 0)) { 1776 1777 if (tcp_lro_rx(&rq->lro, m, 0) == 0) { 1778 rq->lro_pkts_queued ++; 1779 goto post_done; 1780 } 1781 /* If LRO posting fails then try to post to STACK */ 1782 } 1783 #endif 1784 1785 (*sc->ifp->if_input) (sc->ifp, m); 1786 #if defined(INET6) || defined(INET) 1787 post_done: 1788 #endif 1789 /* Update rx stats per queue */ 1790 rq->rx_stats.rx_pkts++; 1791 rq->rx_stats.rx_bytes += cqe->u0.s.pkt_size; 1792 rq->rx_stats.rx_frags += cqe->u0.s.num_fragments; 1793 if (cqe->u0.s.pkt_type == OCE_MULTICAST_PACKET) 1794 rq->rx_stats.rx_mcast_pkts++; 1795 if (cqe->u0.s.pkt_type == OCE_UNICAST_PACKET) 1796 rq->rx_stats.rx_ucast_pkts++; 1797 } 1798 exit: 1799 return; 1800 } 1801 1802 1803 void 1804 oce_discard_rx_comp(struct oce_rq *rq, int num_frags) 1805 { 1806 uint32_t i = 0; 1807 struct oce_packet_desc *pd; 1808 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1809 1810 for (i = 0; i < num_frags; i++) { 1811 if (rq->ring->cidx == rq->ring->pidx) { 1812 device_printf(sc->dev, 1813 "oce_discard_rx_comp: Invalid RX completion - Queue is empty\n"); 1814 return; 1815 } 1816 pd = &rq->pckts[rq->ring->cidx]; 1817 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1818 bus_dmamap_unload(rq->tag, pd->map); 1819 if (pd->mbuf != NULL) { 1820 m_freem(pd->mbuf); 1821 pd->mbuf = NULL; 1822 } 1823 1824 RING_GET(rq->ring, 1); 1825 rq->pending--; 1826 } 1827 } 1828 1829 1830 static int 1831 oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe) 1832 { 1833 struct oce_nic_rx_cqe_v1 *cqe_v1; 1834 int vtp = 0; 1835 1836 if (sc->be3_native) { 1837 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe; 1838 vtp = cqe_v1->u0.s.vlan_tag_present; 1839 } else 1840 vtp = cqe->u0.s.vlan_tag_present; 1841 1842 return vtp; 1843 1844 } 1845 1846 1847 static int 1848 oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe) 1849 { 1850 struct oce_nic_rx_cqe_v1 *cqe_v1; 1851 int port_id = 0; 1852 1853 if (sc->be3_native && (IS_BE(sc) || IS_SH(sc))) { 1854 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe; 1855 port_id = cqe_v1->u0.s.port; 1856 if (sc->port_id != port_id) 1857 return 0; 1858 } else 1859 ;/* For BE3 legacy and Lancer this is dummy */ 1860 1861 return 1; 1862 1863 } 1864 1865 #if defined(INET6) || defined(INET) 1866 void 1867 oce_rx_flush_lro(struct oce_rq *rq) 1868 { 1869 struct lro_ctrl *lro = &rq->lro; 1870 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1871 1872 if (!IF_LRO_ENABLED(sc)) 1873 return; 1874 1875 tcp_lro_flush_all(lro); 1876 rq->lro_pkts_queued = 0; 1877 1878 return; 1879 } 1880 1881 1882 static int 1883 oce_init_lro(POCE_SOFTC sc) 1884 { 1885 struct lro_ctrl *lro = NULL; 1886 int i = 0, rc = 0; 1887 1888 for (i = 0; i < sc->nrqs; i++) { 1889 lro = &sc->rq[i]->lro; 1890 rc = tcp_lro_init(lro); 1891 if (rc != 0) { 1892 device_printf(sc->dev, "LRO init failed\n"); 1893 return rc; 1894 } 1895 lro->ifp = sc->ifp; 1896 } 1897 1898 return rc; 1899 } 1900 1901 1902 void 1903 oce_free_lro(POCE_SOFTC sc) 1904 { 1905 struct lro_ctrl *lro = NULL; 1906 int i = 0; 1907 1908 for (i = 0; i < sc->nrqs; i++) { 1909 lro = &sc->rq[i]->lro; 1910 if (lro) 1911 tcp_lro_free(lro); 1912 } 1913 } 1914 #endif 1915 1916 int 1917 oce_alloc_rx_bufs(struct oce_rq *rq, int count) 1918 { 1919 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1920 int i, in, rc; 1921 struct oce_packet_desc *pd; 1922 bus_dma_segment_t segs[6]; 1923 int nsegs, added = 0; 1924 struct oce_nic_rqe *rqe; 1925 pd_rxulp_db_t rxdb_reg; 1926 uint32_t val = 0; 1927 uint32_t oce_max_rq_posts = 64; 1928 1929 bzero(&rxdb_reg, sizeof(pd_rxulp_db_t)); 1930 for (i = 0; i < count; i++) { 1931 in = (rq->ring->pidx + 1) % OCE_RQ_PACKET_ARRAY_SIZE; 1932 1933 pd = &rq->pckts[rq->ring->pidx]; 1934 pd->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, oce_rq_buf_size); 1935 if (pd->mbuf == NULL) { 1936 device_printf(sc->dev, "mbuf allocation failed, size = %d\n",oce_rq_buf_size); 1937 break; 1938 } 1939 pd->mbuf->m_nextpkt = NULL; 1940 1941 pd->mbuf->m_len = pd->mbuf->m_pkthdr.len = rq->cfg.frag_size; 1942 1943 rc = bus_dmamap_load_mbuf_sg(rq->tag, 1944 pd->map, 1945 pd->mbuf, 1946 segs, &nsegs, BUS_DMA_NOWAIT); 1947 if (rc) { 1948 m_free(pd->mbuf); 1949 device_printf(sc->dev, "bus_dmamap_load_mbuf_sg failed rc = %d\n", rc); 1950 break; 1951 } 1952 1953 if (nsegs != 1) { 1954 i--; 1955 continue; 1956 } 1957 1958 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_PREREAD); 1959 1960 rqe = RING_GET_PRODUCER_ITEM_VA(rq->ring, struct oce_nic_rqe); 1961 rqe->u0.s.frag_pa_hi = ADDR_HI(segs[0].ds_addr); 1962 rqe->u0.s.frag_pa_lo = ADDR_LO(segs[0].ds_addr); 1963 DW_SWAP(u32ptr(rqe), sizeof(struct oce_nic_rqe)); 1964 RING_PUT(rq->ring, 1); 1965 added++; 1966 rq->pending++; 1967 } 1968 oce_max_rq_posts = sc->enable_hwlro ? OCE_HWLRO_MAX_RQ_POSTS : OCE_MAX_RQ_POSTS; 1969 if (added != 0) { 1970 for (i = added / oce_max_rq_posts; i > 0; i--) { 1971 rxdb_reg.bits.num_posted = oce_max_rq_posts; 1972 rxdb_reg.bits.qid = rq->rq_id; 1973 if(rq->islro) { 1974 val |= rq->rq_id & DB_LRO_RQ_ID_MASK; 1975 val |= oce_max_rq_posts << 16; 1976 OCE_WRITE_REG32(sc, db, DB_OFFSET, val); 1977 }else { 1978 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); 1979 } 1980 added -= oce_max_rq_posts; 1981 } 1982 if (added > 0) { 1983 rxdb_reg.bits.qid = rq->rq_id; 1984 rxdb_reg.bits.num_posted = added; 1985 if(rq->islro) { 1986 val |= rq->rq_id & DB_LRO_RQ_ID_MASK; 1987 val |= added << 16; 1988 OCE_WRITE_REG32(sc, db, DB_OFFSET, val); 1989 }else { 1990 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); 1991 } 1992 } 1993 } 1994 1995 return 0; 1996 } 1997 1998 static void 1999 oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq) 2000 { 2001 if (num_cqes) { 2002 oce_arm_cq(sc, rq->cq->cq_id, num_cqes, FALSE); 2003 if(!sc->enable_hwlro) { 2004 if((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) > 1) 2005 oce_alloc_rx_bufs(rq, ((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) - 1)); 2006 }else { 2007 if ((OCE_RQ_PACKET_ARRAY_SIZE -1 - rq->pending) > 64) 2008 oce_alloc_rx_bufs(rq, 64); 2009 } 2010 } 2011 2012 return; 2013 } 2014 2015 uint16_t 2016 oce_rq_handler_lro(void *arg) 2017 { 2018 struct oce_rq *rq = (struct oce_rq *)arg; 2019 struct oce_cq *cq = rq->cq; 2020 POCE_SOFTC sc = rq->parent; 2021 struct nic_hwlro_singleton_cqe *cqe; 2022 struct nic_hwlro_cqe_part2 *cqe2; 2023 int num_cqes = 0; 2024 2025 LOCK(&rq->rx_lock); 2026 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2027 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); 2028 while (cqe->valid) { 2029 if(cqe->cqe_type == 0) { /* singleton cqe */ 2030 /* we should not get singleton cqe after cqe1 on same rq */ 2031 if(rq->cqe_firstpart != NULL) { 2032 device_printf(sc->dev, "Got singleton cqe after cqe1 \n"); 2033 goto exit_rq_handler_lro; 2034 } 2035 if(cqe->error != 0) { 2036 rq->rx_stats.rxcp_err++; 2037 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2038 } 2039 oce_rx_lro(rq, cqe, NULL); 2040 rq->rx_stats.rx_compl++; 2041 cqe->valid = 0; 2042 RING_GET(cq->ring, 1); 2043 num_cqes++; 2044 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2045 break; 2046 }else if(cqe->cqe_type == 0x1) { /* first part */ 2047 /* we should not get cqe1 after cqe1 on same rq */ 2048 if(rq->cqe_firstpart != NULL) { 2049 device_printf(sc->dev, "Got cqe1 after cqe1 \n"); 2050 goto exit_rq_handler_lro; 2051 } 2052 rq->cqe_firstpart = (struct nic_hwlro_cqe_part1 *)cqe; 2053 RING_GET(cq->ring, 1); 2054 }else if(cqe->cqe_type == 0x2) { /* second part */ 2055 cqe2 = (struct nic_hwlro_cqe_part2 *)cqe; 2056 if(cqe2->error != 0) { 2057 rq->rx_stats.rxcp_err++; 2058 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2059 } 2060 /* We should not get cqe2 without cqe1 */ 2061 if(rq->cqe_firstpart == NULL) { 2062 device_printf(sc->dev, "Got cqe2 without cqe1 \n"); 2063 goto exit_rq_handler_lro; 2064 } 2065 oce_rx_lro(rq, (struct nic_hwlro_singleton_cqe *)rq->cqe_firstpart, cqe2); 2066 2067 rq->rx_stats.rx_compl++; 2068 rq->cqe_firstpart->valid = 0; 2069 cqe2->valid = 0; 2070 rq->cqe_firstpart = NULL; 2071 2072 RING_GET(cq->ring, 1); 2073 num_cqes += 2; 2074 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2075 break; 2076 } 2077 2078 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2079 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); 2080 } 2081 oce_check_rx_bufs(sc, num_cqes, rq); 2082 exit_rq_handler_lro: 2083 UNLOCK(&rq->rx_lock); 2084 return 0; 2085 } 2086 2087 /* Handle the Completion Queue for receive */ 2088 uint16_t 2089 oce_rq_handler(void *arg) 2090 { 2091 struct oce_rq *rq = (struct oce_rq *)arg; 2092 struct oce_cq *cq = rq->cq; 2093 POCE_SOFTC sc = rq->parent; 2094 struct oce_nic_rx_cqe *cqe; 2095 int num_cqes = 0; 2096 2097 if(rq->islro) { 2098 oce_rq_handler_lro(arg); 2099 return 0; 2100 } 2101 LOCK(&rq->rx_lock); 2102 bus_dmamap_sync(cq->ring->dma.tag, 2103 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2104 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); 2105 while (cqe->u0.dw[2]) { 2106 DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe)); 2107 2108 if (cqe->u0.s.error == 0) { 2109 oce_rx(rq, cqe); 2110 } else { 2111 rq->rx_stats.rxcp_err++; 2112 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2113 /* Post L3/L4 errors to stack.*/ 2114 oce_rx(rq, cqe); 2115 } 2116 rq->rx_stats.rx_compl++; 2117 cqe->u0.dw[2] = 0; 2118 2119 #if defined(INET6) || defined(INET) 2120 if (IF_LRO_ENABLED(sc) && rq->lro_pkts_queued >= 16) { 2121 oce_rx_flush_lro(rq); 2122 } 2123 #endif 2124 2125 RING_GET(cq->ring, 1); 2126 bus_dmamap_sync(cq->ring->dma.tag, 2127 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2128 cqe = 2129 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); 2130 num_cqes++; 2131 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2132 break; 2133 } 2134 2135 #if defined(INET6) || defined(INET) 2136 if (IF_LRO_ENABLED(sc)) 2137 oce_rx_flush_lro(rq); 2138 #endif 2139 2140 oce_check_rx_bufs(sc, num_cqes, rq); 2141 UNLOCK(&rq->rx_lock); 2142 return 0; 2143 2144 } 2145 2146 2147 2148 2149 /***************************************************************************** 2150 * Helper function prototypes in this file * 2151 *****************************************************************************/ 2152 2153 static int 2154 oce_attach_ifp(POCE_SOFTC sc) 2155 { 2156 2157 sc->ifp = if_alloc(IFT_ETHER); 2158 if (!sc->ifp) 2159 return ENOMEM; 2160 2161 ifmedia_init(&sc->media, IFM_IMASK, oce_media_change, oce_media_status); 2162 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2163 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 2164 2165 sc->ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST; 2166 sc->ifp->if_ioctl = oce_ioctl; 2167 sc->ifp->if_start = oce_start; 2168 sc->ifp->if_init = oce_init; 2169 sc->ifp->if_mtu = ETHERMTU; 2170 sc->ifp->if_softc = sc; 2171 sc->ifp->if_transmit = oce_multiq_start; 2172 sc->ifp->if_qflush = oce_multiq_flush; 2173 2174 if_initname(sc->ifp, 2175 device_get_name(sc->dev), device_get_unit(sc->dev)); 2176 2177 sc->ifp->if_snd.ifq_drv_maxlen = OCE_MAX_TX_DESC - 1; 2178 IFQ_SET_MAXLEN(&sc->ifp->if_snd, sc->ifp->if_snd.ifq_drv_maxlen); 2179 IFQ_SET_READY(&sc->ifp->if_snd); 2180 2181 sc->ifp->if_hwassist = OCE_IF_HWASSIST; 2182 sc->ifp->if_hwassist |= CSUM_TSO; 2183 sc->ifp->if_hwassist |= (CSUM_IP | CSUM_TCP | CSUM_UDP); 2184 2185 sc->ifp->if_capabilities = OCE_IF_CAPABILITIES; 2186 sc->ifp->if_capabilities |= IFCAP_HWCSUM; 2187 sc->ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 2188 2189 #if defined(INET6) || defined(INET) 2190 sc->ifp->if_capabilities |= IFCAP_TSO; 2191 sc->ifp->if_capabilities |= IFCAP_LRO; 2192 sc->ifp->if_capabilities |= IFCAP_VLAN_HWTSO; 2193 #endif 2194 2195 sc->ifp->if_capenable = sc->ifp->if_capabilities; 2196 sc->ifp->if_baudrate = IF_Gbps(10); 2197 2198 sc->ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 2199 sc->ifp->if_hw_tsomaxsegcount = OCE_MAX_TX_ELEMENTS; 2200 sc->ifp->if_hw_tsomaxsegsize = 4096; 2201 2202 ether_ifattach(sc->ifp, sc->macaddr.mac_addr); 2203 2204 return 0; 2205 } 2206 2207 2208 static void 2209 oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag) 2210 { 2211 POCE_SOFTC sc = ifp->if_softc; 2212 2213 if (ifp->if_softc != arg) 2214 return; 2215 if ((vtag == 0) || (vtag > 4095)) 2216 return; 2217 2218 sc->vlan_tag[vtag] = 1; 2219 sc->vlans_added++; 2220 if (sc->vlans_added <= (sc->max_vlans + 1)) 2221 oce_vid_config(sc); 2222 } 2223 2224 2225 static void 2226 oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag) 2227 { 2228 POCE_SOFTC sc = ifp->if_softc; 2229 2230 if (ifp->if_softc != arg) 2231 return; 2232 if ((vtag == 0) || (vtag > 4095)) 2233 return; 2234 2235 sc->vlan_tag[vtag] = 0; 2236 sc->vlans_added--; 2237 oce_vid_config(sc); 2238 } 2239 2240 2241 /* 2242 * A max of 64 vlans can be configured in BE. If the user configures 2243 * more, place the card in vlan promiscuous mode. 2244 */ 2245 static int 2246 oce_vid_config(POCE_SOFTC sc) 2247 { 2248 struct normal_vlan vtags[MAX_VLANFILTER_SIZE]; 2249 uint16_t ntags = 0, i; 2250 int status = 0; 2251 2252 if ((sc->vlans_added <= MAX_VLANFILTER_SIZE) && 2253 (sc->ifp->if_capenable & IFCAP_VLAN_HWFILTER)) { 2254 for (i = 0; i < MAX_VLANS; i++) { 2255 if (sc->vlan_tag[i]) { 2256 vtags[ntags].vtag = i; 2257 ntags++; 2258 } 2259 } 2260 if (ntags) 2261 status = oce_config_vlan(sc, (uint8_t) sc->if_id, 2262 vtags, ntags, 1, 0); 2263 } else 2264 status = oce_config_vlan(sc, (uint8_t) sc->if_id, 2265 NULL, 0, 1, 1); 2266 return status; 2267 } 2268 2269 2270 static void 2271 oce_mac_addr_set(POCE_SOFTC sc) 2272 { 2273 uint32_t old_pmac_id = sc->pmac_id; 2274 int status = 0; 2275 2276 2277 status = bcmp((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr, 2278 sc->macaddr.size_of_struct); 2279 if (!status) 2280 return; 2281 2282 status = oce_mbox_macaddr_add(sc, (uint8_t *)(IF_LLADDR(sc->ifp)), 2283 sc->if_id, &sc->pmac_id); 2284 if (!status) { 2285 status = oce_mbox_macaddr_del(sc, sc->if_id, old_pmac_id); 2286 bcopy((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr, 2287 sc->macaddr.size_of_struct); 2288 } 2289 if (status) 2290 device_printf(sc->dev, "Failed update macaddress\n"); 2291 2292 } 2293 2294 2295 static int 2296 oce_handle_passthrough(struct ifnet *ifp, caddr_t data) 2297 { 2298 POCE_SOFTC sc = ifp->if_softc; 2299 struct ifreq *ifr = (struct ifreq *)data; 2300 int rc = ENXIO; 2301 char cookie[32] = {0}; 2302 void *priv_data = ifr_data_get_ptr(ifr); 2303 void *ioctl_ptr; 2304 uint32_t req_size; 2305 struct mbx_hdr req; 2306 OCE_DMA_MEM dma_mem; 2307 struct mbx_common_get_cntl_attr *fw_cmd; 2308 2309 if (copyin(priv_data, cookie, strlen(IOCTL_COOKIE))) 2310 return EFAULT; 2311 2312 if (memcmp(cookie, IOCTL_COOKIE, strlen(IOCTL_COOKIE))) 2313 return EINVAL; 2314 2315 ioctl_ptr = (char *)priv_data + strlen(IOCTL_COOKIE); 2316 if (copyin(ioctl_ptr, &req, sizeof(struct mbx_hdr))) 2317 return EFAULT; 2318 2319 req_size = le32toh(req.u0.req.request_length); 2320 if (req_size > 65536) 2321 return EINVAL; 2322 2323 req_size += sizeof(struct mbx_hdr); 2324 rc = oce_dma_alloc(sc, req_size, &dma_mem, 0); 2325 if (rc) 2326 return ENOMEM; 2327 2328 if (copyin(ioctl_ptr, OCE_DMAPTR(&dma_mem,char), req_size)) { 2329 rc = EFAULT; 2330 goto dma_free; 2331 } 2332 2333 rc = oce_pass_through_mbox(sc, &dma_mem, req_size); 2334 if (rc) { 2335 rc = EIO; 2336 goto dma_free; 2337 } 2338 2339 if (copyout(OCE_DMAPTR(&dma_mem,char), ioctl_ptr, req_size)) 2340 rc = EFAULT; 2341 2342 /* 2343 firmware is filling all the attributes for this ioctl except 2344 the driver version..so fill it 2345 */ 2346 if(req.u0.rsp.opcode == OPCODE_COMMON_GET_CNTL_ATTRIBUTES) { 2347 fw_cmd = (struct mbx_common_get_cntl_attr *) ioctl_ptr; 2348 strncpy(fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str, 2349 COMPONENT_REVISION, strlen(COMPONENT_REVISION)); 2350 } 2351 2352 dma_free: 2353 oce_dma_free(sc, &dma_mem); 2354 return rc; 2355 2356 } 2357 2358 static void 2359 oce_eqd_set_periodic(POCE_SOFTC sc) 2360 { 2361 struct oce_set_eqd set_eqd[OCE_MAX_EQ]; 2362 struct oce_aic_obj *aic; 2363 struct oce_eq *eqo; 2364 uint64_t now = 0, delta; 2365 int eqd, i, num = 0; 2366 uint32_t tx_reqs = 0, rxpkts = 0, pps; 2367 struct oce_wq *wq; 2368 struct oce_rq *rq; 2369 2370 #define ticks_to_msecs(t) (1000 * (t) / hz) 2371 2372 for (i = 0 ; i < sc->neqs; i++) { 2373 eqo = sc->eq[i]; 2374 aic = &sc->aic_obj[i]; 2375 /* When setting the static eq delay from the user space */ 2376 if (!aic->enable) { 2377 if (aic->ticks) 2378 aic->ticks = 0; 2379 eqd = aic->et_eqd; 2380 goto modify_eqd; 2381 } 2382 2383 if (i == 0) { 2384 rq = sc->rq[0]; 2385 rxpkts = rq->rx_stats.rx_pkts; 2386 } else 2387 rxpkts = 0; 2388 if (i + 1 < sc->nrqs) { 2389 rq = sc->rq[i + 1]; 2390 rxpkts += rq->rx_stats.rx_pkts; 2391 } 2392 if (i < sc->nwqs) { 2393 wq = sc->wq[i]; 2394 tx_reqs = wq->tx_stats.tx_reqs; 2395 } else 2396 tx_reqs = 0; 2397 now = ticks; 2398 2399 if (!aic->ticks || now < aic->ticks || 2400 rxpkts < aic->prev_rxpkts || tx_reqs < aic->prev_txreqs) { 2401 aic->prev_rxpkts = rxpkts; 2402 aic->prev_txreqs = tx_reqs; 2403 aic->ticks = now; 2404 continue; 2405 } 2406 2407 delta = ticks_to_msecs(now - aic->ticks); 2408 2409 pps = (((uint32_t)(rxpkts - aic->prev_rxpkts) * 1000) / delta) + 2410 (((uint32_t)(tx_reqs - aic->prev_txreqs) * 1000) / delta); 2411 eqd = (pps / 15000) << 2; 2412 if (eqd < 8) 2413 eqd = 0; 2414 2415 /* Make sure that the eq delay is in the known range */ 2416 eqd = min(eqd, aic->max_eqd); 2417 eqd = max(eqd, aic->min_eqd); 2418 2419 aic->prev_rxpkts = rxpkts; 2420 aic->prev_txreqs = tx_reqs; 2421 aic->ticks = now; 2422 2423 modify_eqd: 2424 if (eqd != aic->cur_eqd) { 2425 set_eqd[num].delay_multiplier = (eqd * 65)/100; 2426 set_eqd[num].eq_id = eqo->eq_id; 2427 aic->cur_eqd = eqd; 2428 num++; 2429 } 2430 } 2431 2432 /* Is there atleast one eq that needs to be modified? */ 2433 for(i = 0; i < num; i += 8) { 2434 if((num - i) >=8 ) 2435 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], 8); 2436 else 2437 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], (num - i)); 2438 } 2439 2440 } 2441 2442 static void oce_detect_hw_error(POCE_SOFTC sc) 2443 { 2444 2445 uint32_t ue_low = 0, ue_high = 0, ue_low_mask = 0, ue_high_mask = 0; 2446 uint32_t sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0; 2447 uint32_t i; 2448 2449 if (sc->hw_error) 2450 return; 2451 2452 if (IS_XE201(sc)) { 2453 sliport_status = OCE_READ_REG32(sc, db, SLIPORT_STATUS_OFFSET); 2454 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2455 sliport_err1 = OCE_READ_REG32(sc, db, SLIPORT_ERROR1_OFFSET); 2456 sliport_err2 = OCE_READ_REG32(sc, db, SLIPORT_ERROR2_OFFSET); 2457 } 2458 } else { 2459 ue_low = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW); 2460 ue_high = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HIGH); 2461 ue_low_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW_MASK); 2462 ue_high_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HI_MASK); 2463 2464 ue_low = (ue_low & ~ue_low_mask); 2465 ue_high = (ue_high & ~ue_high_mask); 2466 } 2467 2468 /* On certain platforms BE hardware can indicate spurious UEs. 2469 * Allow the h/w to stop working completely in case of a real UE. 2470 * Hence not setting the hw_error for UE detection. 2471 */ 2472 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2473 sc->hw_error = TRUE; 2474 device_printf(sc->dev, "Error detected in the card\n"); 2475 } 2476 2477 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2478 device_printf(sc->dev, 2479 "ERR: sliport status 0x%x\n", sliport_status); 2480 device_printf(sc->dev, 2481 "ERR: sliport error1 0x%x\n", sliport_err1); 2482 device_printf(sc->dev, 2483 "ERR: sliport error2 0x%x\n", sliport_err2); 2484 } 2485 2486 if (ue_low) { 2487 for (i = 0; ue_low; ue_low >>= 1, i++) { 2488 if (ue_low & 1) 2489 device_printf(sc->dev, "UE: %s bit set\n", 2490 ue_status_low_desc[i]); 2491 } 2492 } 2493 2494 if (ue_high) { 2495 for (i = 0; ue_high; ue_high >>= 1, i++) { 2496 if (ue_high & 1) 2497 device_printf(sc->dev, "UE: %s bit set\n", 2498 ue_status_hi_desc[i]); 2499 } 2500 } 2501 2502 } 2503 2504 2505 static void 2506 oce_local_timer(void *arg) 2507 { 2508 POCE_SOFTC sc = arg; 2509 int i = 0; 2510 2511 oce_detect_hw_error(sc); 2512 oce_refresh_nic_stats(sc); 2513 oce_refresh_queue_stats(sc); 2514 oce_mac_addr_set(sc); 2515 2516 /* TX Watch Dog*/ 2517 for (i = 0; i < sc->nwqs; i++) 2518 oce_tx_restart(sc, sc->wq[i]); 2519 2520 /* calculate and set the eq delay for optimal interrupt rate */ 2521 if (IS_BE(sc) || IS_SH(sc)) 2522 oce_eqd_set_periodic(sc); 2523 2524 callout_reset(&sc->timer, hz, oce_local_timer, sc); 2525 } 2526 2527 static void 2528 oce_tx_compl_clean(POCE_SOFTC sc) 2529 { 2530 struct oce_wq *wq; 2531 int i = 0, timeo = 0, num_wqes = 0; 2532 int pending_txqs = sc->nwqs; 2533 2534 /* Stop polling for compls when HW has been silent for 10ms or 2535 * hw_error or no outstanding completions expected 2536 */ 2537 do { 2538 pending_txqs = sc->nwqs; 2539 2540 for_all_wq_queues(sc, wq, i) { 2541 num_wqes = oce_wq_handler(wq); 2542 2543 if(num_wqes) 2544 timeo = 0; 2545 2546 if(!wq->ring->num_used) 2547 pending_txqs--; 2548 } 2549 2550 if (pending_txqs == 0 || ++timeo > 10 || sc->hw_error) 2551 break; 2552 2553 DELAY(1000); 2554 } while (TRUE); 2555 2556 for_all_wq_queues(sc, wq, i) { 2557 while(wq->ring->num_used) { 2558 LOCK(&wq->tx_compl_lock); 2559 oce_process_tx_completion(wq); 2560 UNLOCK(&wq->tx_compl_lock); 2561 } 2562 } 2563 2564 } 2565 2566 /* NOTE : This should only be called holding 2567 * DEVICE_LOCK. 2568 */ 2569 static void 2570 oce_if_deactivate(POCE_SOFTC sc) 2571 { 2572 int i; 2573 struct oce_rq *rq; 2574 struct oce_wq *wq; 2575 struct oce_eq *eq; 2576 2577 sc->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 2578 2579 oce_tx_compl_clean(sc); 2580 2581 /* Stop intrs and finish any bottom halves pending */ 2582 oce_hw_intr_disable(sc); 2583 2584 /* Since taskqueue_drain takes a Gaint Lock, We should not acquire 2585 any other lock. So unlock device lock and require after 2586 completing taskqueue_drain. 2587 */ 2588 UNLOCK(&sc->dev_lock); 2589 for (i = 0; i < sc->intr_count; i++) { 2590 if (sc->intrs[i].tq != NULL) { 2591 taskqueue_drain(sc->intrs[i].tq, &sc->intrs[i].task); 2592 } 2593 } 2594 LOCK(&sc->dev_lock); 2595 2596 /* Delete RX queue in card with flush param */ 2597 oce_stop_rx(sc); 2598 2599 /* Invalidate any pending cq and eq entries*/ 2600 for_all_evnt_queues(sc, eq, i) 2601 oce_drain_eq(eq); 2602 for_all_rq_queues(sc, rq, i) 2603 oce_drain_rq_cq(rq); 2604 for_all_wq_queues(sc, wq, i) 2605 oce_drain_wq_cq(wq); 2606 2607 /* But still we need to get MCC aync events. 2608 So enable intrs and also arm first EQ 2609 */ 2610 oce_hw_intr_enable(sc); 2611 oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE); 2612 2613 DELAY(10); 2614 } 2615 2616 2617 static void 2618 oce_if_activate(POCE_SOFTC sc) 2619 { 2620 struct oce_eq *eq; 2621 struct oce_rq *rq; 2622 struct oce_wq *wq; 2623 int i, rc = 0; 2624 2625 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; 2626 2627 oce_hw_intr_disable(sc); 2628 2629 oce_start_rx(sc); 2630 2631 for_all_rq_queues(sc, rq, i) { 2632 rc = oce_start_rq(rq); 2633 if (rc) 2634 device_printf(sc->dev, "Unable to start RX\n"); 2635 } 2636 2637 for_all_wq_queues(sc, wq, i) { 2638 rc = oce_start_wq(wq); 2639 if (rc) 2640 device_printf(sc->dev, "Unable to start TX\n"); 2641 } 2642 2643 2644 for_all_evnt_queues(sc, eq, i) 2645 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); 2646 2647 oce_hw_intr_enable(sc); 2648 2649 } 2650 2651 static void 2652 process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe) 2653 { 2654 /* Update Link status */ 2655 if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) == 2656 ASYNC_EVENT_LINK_UP) { 2657 sc->link_status = ASYNC_EVENT_LINK_UP; 2658 if_link_state_change(sc->ifp, LINK_STATE_UP); 2659 } else { 2660 sc->link_status = ASYNC_EVENT_LINK_DOWN; 2661 if_link_state_change(sc->ifp, LINK_STATE_DOWN); 2662 } 2663 } 2664 2665 2666 static void oce_async_grp5_osbmc_process(POCE_SOFTC sc, 2667 struct oce_async_evt_grp5_os2bmc *evt) 2668 { 2669 DW_SWAP(evt, sizeof(struct oce_async_evt_grp5_os2bmc)); 2670 if (evt->u.s.mgmt_enable) 2671 sc->flags |= OCE_FLAGS_OS2BMC; 2672 else 2673 return; 2674 2675 sc->bmc_filt_mask = evt->u.s.arp_filter; 2676 sc->bmc_filt_mask |= (evt->u.s.dhcp_client_filt << 1); 2677 sc->bmc_filt_mask |= (evt->u.s.dhcp_server_filt << 2); 2678 sc->bmc_filt_mask |= (evt->u.s.net_bios_filt << 3); 2679 sc->bmc_filt_mask |= (evt->u.s.bcast_filt << 4); 2680 sc->bmc_filt_mask |= (evt->u.s.ipv6_nbr_filt << 5); 2681 sc->bmc_filt_mask |= (evt->u.s.ipv6_ra_filt << 6); 2682 sc->bmc_filt_mask |= (evt->u.s.ipv6_ras_filt << 7); 2683 sc->bmc_filt_mask |= (evt->u.s.mcast_filt << 8); 2684 } 2685 2686 2687 static void oce_process_grp5_events(POCE_SOFTC sc, struct oce_mq_cqe *cqe) 2688 { 2689 struct oce_async_event_grp5_pvid_state *gcqe; 2690 struct oce_async_evt_grp5_os2bmc *bmccqe; 2691 2692 switch (cqe->u0.s.async_type) { 2693 case ASYNC_EVENT_PVID_STATE: 2694 /* GRP5 PVID */ 2695 gcqe = (struct oce_async_event_grp5_pvid_state *)cqe; 2696 if (gcqe->enabled) 2697 sc->pvid = gcqe->tag & VLAN_VID_MASK; 2698 else 2699 sc->pvid = 0; 2700 break; 2701 case ASYNC_EVENT_OS2BMC: 2702 bmccqe = (struct oce_async_evt_grp5_os2bmc *)cqe; 2703 oce_async_grp5_osbmc_process(sc, bmccqe); 2704 break; 2705 default: 2706 break; 2707 } 2708 } 2709 2710 /* Handle the Completion Queue for the Mailbox/Async notifications */ 2711 uint16_t 2712 oce_mq_handler(void *arg) 2713 { 2714 struct oce_mq *mq = (struct oce_mq *)arg; 2715 POCE_SOFTC sc = mq->parent; 2716 struct oce_cq *cq = mq->cq; 2717 int num_cqes = 0, evt_type = 0, optype = 0; 2718 struct oce_mq_cqe *cqe; 2719 struct oce_async_cqe_link_state *acqe; 2720 struct oce_async_event_qnq *dbgcqe; 2721 2722 2723 bus_dmamap_sync(cq->ring->dma.tag, 2724 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2725 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe); 2726 2727 while (cqe->u0.dw[3]) { 2728 DW_SWAP((uint32_t *) cqe, sizeof(oce_mq_cqe)); 2729 if (cqe->u0.s.async_event) { 2730 evt_type = cqe->u0.s.event_type; 2731 optype = cqe->u0.s.async_type; 2732 if (evt_type == ASYNC_EVENT_CODE_LINK_STATE) { 2733 /* Link status evt */ 2734 acqe = (struct oce_async_cqe_link_state *)cqe; 2735 process_link_state(sc, acqe); 2736 } else if (evt_type == ASYNC_EVENT_GRP5) { 2737 oce_process_grp5_events(sc, cqe); 2738 } else if (evt_type == ASYNC_EVENT_CODE_DEBUG && 2739 optype == ASYNC_EVENT_DEBUG_QNQ) { 2740 dbgcqe = (struct oce_async_event_qnq *)cqe; 2741 if(dbgcqe->valid) 2742 sc->qnqid = dbgcqe->vlan_tag; 2743 sc->qnq_debug_event = TRUE; 2744 } 2745 } 2746 cqe->u0.dw[3] = 0; 2747 RING_GET(cq->ring, 1); 2748 bus_dmamap_sync(cq->ring->dma.tag, 2749 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2750 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe); 2751 num_cqes++; 2752 } 2753 2754 if (num_cqes) 2755 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); 2756 2757 return 0; 2758 } 2759 2760 2761 static void 2762 setup_max_queues_want(POCE_SOFTC sc) 2763 { 2764 /* Check if it is FLEX machine. Is so dont use RSS */ 2765 if ((sc->function_mode & FNM_FLEX10_MODE) || 2766 (sc->function_mode & FNM_UMC_MODE) || 2767 (sc->function_mode & FNM_VNIC_MODE) || 2768 (!is_rss_enabled(sc)) || 2769 IS_BE2(sc)) { 2770 sc->nrqs = 1; 2771 sc->nwqs = 1; 2772 } else { 2773 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1; 2774 sc->nwqs = MIN(OCE_NCPUS, sc->nrssqs); 2775 } 2776 2777 if (IS_BE2(sc) && is_rss_enabled(sc)) 2778 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1; 2779 } 2780 2781 2782 static void 2783 update_queues_got(POCE_SOFTC sc) 2784 { 2785 if (is_rss_enabled(sc)) { 2786 sc->nrqs = sc->intr_count + 1; 2787 sc->nwqs = sc->intr_count; 2788 } else { 2789 sc->nrqs = 1; 2790 sc->nwqs = 1; 2791 } 2792 2793 if (IS_BE2(sc)) 2794 sc->nwqs = 1; 2795 } 2796 2797 static int 2798 oce_check_ipv6_ext_hdr(struct mbuf *m) 2799 { 2800 struct ether_header *eh = mtod(m, struct ether_header *); 2801 caddr_t m_datatemp = m->m_data; 2802 2803 if (eh->ether_type == htons(ETHERTYPE_IPV6)) { 2804 m->m_data += sizeof(struct ether_header); 2805 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); 2806 2807 if((ip6->ip6_nxt != IPPROTO_TCP) && \ 2808 (ip6->ip6_nxt != IPPROTO_UDP)){ 2809 struct ip6_ext *ip6e = NULL; 2810 m->m_data += sizeof(struct ip6_hdr); 2811 2812 ip6e = (struct ip6_ext *) mtod(m, struct ip6_ext *); 2813 if(ip6e->ip6e_len == 0xff) { 2814 m->m_data = m_datatemp; 2815 return TRUE; 2816 } 2817 } 2818 m->m_data = m_datatemp; 2819 } 2820 return FALSE; 2821 } 2822 2823 static int 2824 is_be3_a1(POCE_SOFTC sc) 2825 { 2826 if((sc->flags & OCE_FLAGS_BE3) && ((sc->asic_revision & 0xFF) < 2)) { 2827 return TRUE; 2828 } 2829 return FALSE; 2830 } 2831 2832 static struct mbuf * 2833 oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete) 2834 { 2835 uint16_t vlan_tag = 0; 2836 2837 if(!M_WRITABLE(m)) 2838 return NULL; 2839 2840 /* Embed vlan tag in the packet if it is not part of it */ 2841 if(m->m_flags & M_VLANTAG) { 2842 vlan_tag = EVL_VLANOFTAG(m->m_pkthdr.ether_vtag); 2843 m->m_flags &= ~M_VLANTAG; 2844 } 2845 2846 /* if UMC, ignore vlan tag insertion and instead insert pvid */ 2847 if(sc->pvid) { 2848 if(!vlan_tag) 2849 vlan_tag = sc->pvid; 2850 if (complete) 2851 *complete = FALSE; 2852 } 2853 2854 if(vlan_tag) { 2855 m = ether_vlanencap(m, vlan_tag); 2856 } 2857 2858 if(sc->qnqid) { 2859 m = ether_vlanencap(m, sc->qnqid); 2860 2861 if (complete) 2862 *complete = FALSE; 2863 } 2864 return m; 2865 } 2866 2867 static int 2868 oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m) 2869 { 2870 if(is_be3_a1(sc) && IS_QNQ_OR_UMC(sc) && \ 2871 oce_check_ipv6_ext_hdr(m)) { 2872 return TRUE; 2873 } 2874 return FALSE; 2875 } 2876 2877 static void 2878 oce_get_config(POCE_SOFTC sc) 2879 { 2880 int rc = 0; 2881 uint32_t max_rss = 0; 2882 2883 if ((IS_BE(sc) || IS_SH(sc)) && (!sc->be3_native)) 2884 max_rss = OCE_LEGACY_MODE_RSS; 2885 else 2886 max_rss = OCE_MAX_RSS; 2887 2888 if (!IS_BE(sc)) { 2889 rc = oce_get_profile_config(sc, max_rss); 2890 if (rc) { 2891 sc->nwqs = OCE_MAX_WQ; 2892 sc->nrssqs = max_rss; 2893 sc->nrqs = sc->nrssqs + 1; 2894 } 2895 } 2896 else { /* For BE3 don't rely on fw for determining the resources */ 2897 sc->nrssqs = max_rss; 2898 sc->nrqs = sc->nrssqs + 1; 2899 sc->nwqs = OCE_MAX_WQ; 2900 sc->max_vlans = MAX_VLANFILTER_SIZE; 2901 } 2902 } 2903 2904 static void 2905 oce_rdma_close(void) 2906 { 2907 if (oce_rdma_if != NULL) { 2908 oce_rdma_if = NULL; 2909 } 2910 } 2911 2912 static void 2913 oce_get_mac_addr(POCE_SOFTC sc, uint8_t *macaddr) 2914 { 2915 memcpy(macaddr, sc->macaddr.mac_addr, 6); 2916 } 2917 2918 int 2919 oce_register_rdma(POCE_RDMA_INFO rdma_info, POCE_RDMA_IF rdma_if) 2920 { 2921 POCE_SOFTC sc; 2922 struct oce_dev_info di; 2923 int i; 2924 2925 if ((rdma_info == NULL) || (rdma_if == NULL)) { 2926 return -EINVAL; 2927 } 2928 2929 if ((rdma_info->size != OCE_RDMA_INFO_SIZE) || 2930 (rdma_if->size != OCE_RDMA_IF_SIZE)) { 2931 return -ENXIO; 2932 } 2933 2934 rdma_info->close = oce_rdma_close; 2935 rdma_info->mbox_post = oce_mbox_post; 2936 rdma_info->common_req_hdr_init = mbx_common_req_hdr_init; 2937 rdma_info->get_mac_addr = oce_get_mac_addr; 2938 2939 oce_rdma_if = rdma_if; 2940 2941 sc = softc_head; 2942 while (sc != NULL) { 2943 if (oce_rdma_if->announce != NULL) { 2944 memset(&di, 0, sizeof(di)); 2945 di.dev = sc->dev; 2946 di.softc = sc; 2947 di.ifp = sc->ifp; 2948 di.db_bhandle = sc->db_bhandle; 2949 di.db_btag = sc->db_btag; 2950 di.db_page_size = 4096; 2951 if (sc->flags & OCE_FLAGS_USING_MSIX) { 2952 di.intr_mode = OCE_INTERRUPT_MODE_MSIX; 2953 } else if (sc->flags & OCE_FLAGS_USING_MSI) { 2954 di.intr_mode = OCE_INTERRUPT_MODE_MSI; 2955 } else { 2956 di.intr_mode = OCE_INTERRUPT_MODE_INTX; 2957 } 2958 di.dev_family = OCE_GEN2_FAMILY; // fixme: must detect skyhawk 2959 if (di.intr_mode != OCE_INTERRUPT_MODE_INTX) { 2960 di.msix.num_vectors = sc->intr_count + sc->roce_intr_count; 2961 di.msix.start_vector = sc->intr_count; 2962 for (i=0; i<di.msix.num_vectors; i++) { 2963 di.msix.vector_list[i] = sc->intrs[i].vector; 2964 } 2965 } else { 2966 } 2967 memcpy(di.mac_addr, sc->macaddr.mac_addr, 6); 2968 di.vendor_id = pci_get_vendor(sc->dev); 2969 di.dev_id = pci_get_device(sc->dev); 2970 2971 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 2972 di.flags |= OCE_RDMA_INFO_RDMA_SUPPORTED; 2973 } 2974 2975 rdma_if->announce(&di); 2976 sc = sc->next; 2977 } 2978 } 2979 2980 return 0; 2981 } 2982 2983 static void 2984 oce_read_env_variables( POCE_SOFTC sc ) 2985 { 2986 char *value = NULL; 2987 int rc = 0; 2988 2989 /* read if user wants to enable hwlro or swlro */ 2990 //value = getenv("oce_enable_hwlro"); 2991 if(value && IS_SH(sc)) { 2992 sc->enable_hwlro = strtol(value, NULL, 10); 2993 if(sc->enable_hwlro) { 2994 rc = oce_mbox_nic_query_lro_capabilities(sc, NULL, NULL); 2995 if(rc) { 2996 device_printf(sc->dev, "no hardware lro support\n"); 2997 device_printf(sc->dev, "software lro enabled\n"); 2998 sc->enable_hwlro = 0; 2999 }else { 3000 device_printf(sc->dev, "hardware lro enabled\n"); 3001 oce_max_rsp_handled = 32; 3002 } 3003 }else { 3004 device_printf(sc->dev, "software lro enabled\n"); 3005 } 3006 }else { 3007 sc->enable_hwlro = 0; 3008 } 3009 3010 /* read mbuf size */ 3011 //value = getenv("oce_rq_buf_size"); 3012 if(value && IS_SH(sc)) { 3013 oce_rq_buf_size = strtol(value, NULL, 10); 3014 switch(oce_rq_buf_size) { 3015 case 2048: 3016 case 4096: 3017 case 9216: 3018 case 16384: 3019 break; 3020 3021 default: 3022 device_printf(sc->dev, " Supported oce_rq_buf_size values are 2K, 4K, 9K, 16K \n"); 3023 oce_rq_buf_size = 2048; 3024 } 3025 } 3026 3027 return; 3028 } 3029