1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (C) 2013 Emulex 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Emulex Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 * 33 * Contact Information: 34 * freebsd-drivers@emulex.com 35 * 36 * Emulex 37 * 3333 Susan Street 38 * Costa Mesa, CA 92626 39 */ 40 41 /* $FreeBSD$ */ 42 43 #include "opt_inet6.h" 44 #include "opt_inet.h" 45 46 #include "oce_if.h" 47 #include "oce_user.h" 48 49 #define is_tso_pkt(m) (m->m_pkthdr.csum_flags & CSUM_TSO) 50 51 /* UE Status Low CSR */ 52 static char *ue_status_low_desc[] = { 53 "CEV", 54 "CTX", 55 "DBUF", 56 "ERX", 57 "Host", 58 "MPU", 59 "NDMA", 60 "PTC ", 61 "RDMA ", 62 "RXF ", 63 "RXIPS ", 64 "RXULP0 ", 65 "RXULP1 ", 66 "RXULP2 ", 67 "TIM ", 68 "TPOST ", 69 "TPRE ", 70 "TXIPS ", 71 "TXULP0 ", 72 "TXULP1 ", 73 "UC ", 74 "WDMA ", 75 "TXULP2 ", 76 "HOST1 ", 77 "P0_OB_LINK ", 78 "P1_OB_LINK ", 79 "HOST_GPIO ", 80 "MBOX ", 81 "AXGMAC0", 82 "AXGMAC1", 83 "JTAG", 84 "MPU_INTPEND" 85 }; 86 87 /* UE Status High CSR */ 88 static char *ue_status_hi_desc[] = { 89 "LPCMEMHOST", 90 "MGMT_MAC", 91 "PCS0ONLINE", 92 "MPU_IRAM", 93 "PCS1ONLINE", 94 "PCTL0", 95 "PCTL1", 96 "PMEM", 97 "RR", 98 "TXPB", 99 "RXPP", 100 "XAUI", 101 "TXP", 102 "ARM", 103 "IPC", 104 "HOST2", 105 "HOST3", 106 "HOST4", 107 "HOST5", 108 "HOST6", 109 "HOST7", 110 "HOST8", 111 "HOST9", 112 "NETC", 113 "Unknown", 114 "Unknown", 115 "Unknown", 116 "Unknown", 117 "Unknown", 118 "Unknown", 119 "Unknown", 120 "Unknown" 121 }; 122 123 struct oce_common_cqe_info{ 124 uint8_t vtp:1; 125 uint8_t l4_cksum_pass:1; 126 uint8_t ip_cksum_pass:1; 127 uint8_t ipv6_frame:1; 128 uint8_t qnq:1; 129 uint8_t rsvd:3; 130 uint8_t num_frags; 131 uint16_t pkt_size; 132 uint16_t vtag; 133 }; 134 135 136 /* Driver entry points prototypes */ 137 static int oce_probe(device_t dev); 138 static int oce_attach(device_t dev); 139 static int oce_detach(device_t dev); 140 static int oce_shutdown(device_t dev); 141 static int oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data); 142 static void oce_init(void *xsc); 143 static int oce_multiq_start(struct ifnet *ifp, struct mbuf *m); 144 static void oce_multiq_flush(struct ifnet *ifp); 145 146 /* Driver interrupt routines protypes */ 147 static void oce_intr(void *arg, int pending); 148 static int oce_setup_intr(POCE_SOFTC sc); 149 static int oce_fast_isr(void *arg); 150 static int oce_alloc_intr(POCE_SOFTC sc, int vector, 151 void (*isr) (void *arg, int pending)); 152 153 /* Media callbacks prototypes */ 154 static void oce_media_status(struct ifnet *ifp, struct ifmediareq *req); 155 static int oce_media_change(struct ifnet *ifp); 156 157 /* Transmit routines prototypes */ 158 static int oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index); 159 static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq); 160 static void oce_process_tx_completion(struct oce_wq *wq); 161 static int oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, 162 struct oce_wq *wq); 163 164 /* Receive routines prototypes */ 165 static int oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe); 166 static int oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe); 167 static void oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe); 168 static void oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq); 169 static uint16_t oce_rq_handler_lro(void *arg); 170 static void oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2); 171 static void oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2); 172 static void oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m); 173 174 /* Helper function prototypes in this file */ 175 static int oce_attach_ifp(POCE_SOFTC sc); 176 static void oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag); 177 static void oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag); 178 static int oce_vid_config(POCE_SOFTC sc); 179 static void oce_mac_addr_set(POCE_SOFTC sc); 180 static int oce_handle_passthrough(struct ifnet *ifp, caddr_t data); 181 static void oce_local_timer(void *arg); 182 static void oce_if_deactivate(POCE_SOFTC sc); 183 static void oce_if_activate(POCE_SOFTC sc); 184 static void setup_max_queues_want(POCE_SOFTC sc); 185 static void update_queues_got(POCE_SOFTC sc); 186 static void process_link_state(POCE_SOFTC sc, 187 struct oce_async_cqe_link_state *acqe); 188 static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m); 189 static void oce_get_config(POCE_SOFTC sc); 190 static struct mbuf *oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete); 191 static void oce_read_env_variables(POCE_SOFTC sc); 192 193 194 /* IP specific */ 195 #if defined(INET6) || defined(INET) 196 static int oce_init_lro(POCE_SOFTC sc); 197 static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp); 198 #endif 199 200 static device_method_t oce_dispatch[] = { 201 DEVMETHOD(device_probe, oce_probe), 202 DEVMETHOD(device_attach, oce_attach), 203 DEVMETHOD(device_detach, oce_detach), 204 DEVMETHOD(device_shutdown, oce_shutdown), 205 206 DEVMETHOD_END 207 }; 208 209 static driver_t oce_driver = { 210 "oce", 211 oce_dispatch, 212 sizeof(OCE_SOFTC) 213 }; 214 static devclass_t oce_devclass; 215 216 217 /* global vars */ 218 const char component_revision[32] = {"///" COMPONENT_REVISION "///"}; 219 220 /* Module capabilites and parameters */ 221 uint32_t oce_max_rsp_handled = OCE_MAX_RSP_HANDLED; 222 uint32_t oce_enable_rss = OCE_MODCAP_RSS; 223 uint32_t oce_rq_buf_size = 2048; 224 225 TUNABLE_INT("hw.oce.max_rsp_handled", &oce_max_rsp_handled); 226 TUNABLE_INT("hw.oce.enable_rss", &oce_enable_rss); 227 228 229 /* Supported devices table */ 230 static uint32_t supportedDevices[] = { 231 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE2, 232 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE3, 233 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_BE3, 234 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201, 235 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201_VF, 236 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_SH 237 }; 238 239 240 DRIVER_MODULE(oce, pci, oce_driver, oce_devclass, 0, 0); 241 MODULE_PNP_INFO("W32:vendor/device", pci, oce, supportedDevices, 242 nitems(supportedDevices)); 243 MODULE_DEPEND(oce, pci, 1, 1, 1); 244 MODULE_DEPEND(oce, ether, 1, 1, 1); 245 MODULE_VERSION(oce, 1); 246 247 248 POCE_SOFTC softc_head = NULL; 249 POCE_SOFTC softc_tail = NULL; 250 251 struct oce_rdma_if *oce_rdma_if = NULL; 252 253 /***************************************************************************** 254 * Driver entry points functions * 255 *****************************************************************************/ 256 257 static int 258 oce_probe(device_t dev) 259 { 260 uint16_t vendor = 0; 261 uint16_t device = 0; 262 int i = 0; 263 char str[256] = {0}; 264 POCE_SOFTC sc; 265 266 sc = device_get_softc(dev); 267 bzero(sc, sizeof(OCE_SOFTC)); 268 sc->dev = dev; 269 270 vendor = pci_get_vendor(dev); 271 device = pci_get_device(dev); 272 273 for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint32_t)); i++) { 274 if (vendor == ((supportedDevices[i] >> 16) & 0xffff)) { 275 if (device == (supportedDevices[i] & 0xffff)) { 276 sprintf(str, "%s:%s", "Emulex CNA NIC function", 277 component_revision); 278 device_set_desc_copy(dev, str); 279 280 switch (device) { 281 case PCI_PRODUCT_BE2: 282 sc->flags |= OCE_FLAGS_BE2; 283 break; 284 case PCI_PRODUCT_BE3: 285 sc->flags |= OCE_FLAGS_BE3; 286 break; 287 case PCI_PRODUCT_XE201: 288 case PCI_PRODUCT_XE201_VF: 289 sc->flags |= OCE_FLAGS_XE201; 290 break; 291 case PCI_PRODUCT_SH: 292 sc->flags |= OCE_FLAGS_SH; 293 break; 294 default: 295 return ENXIO; 296 } 297 return BUS_PROBE_DEFAULT; 298 } 299 } 300 } 301 302 return ENXIO; 303 } 304 305 306 static int 307 oce_attach(device_t dev) 308 { 309 POCE_SOFTC sc; 310 int rc = 0; 311 312 sc = device_get_softc(dev); 313 314 rc = oce_hw_pci_alloc(sc); 315 if (rc) 316 return rc; 317 318 sc->tx_ring_size = OCE_TX_RING_SIZE; 319 sc->rx_ring_size = OCE_RX_RING_SIZE; 320 /* receive fragment size should be multiple of 2K */ 321 sc->rq_frag_size = ((oce_rq_buf_size / 2048) * 2048); 322 sc->flow_control = OCE_DEFAULT_FLOW_CONTROL; 323 sc->promisc = OCE_DEFAULT_PROMISCUOUS; 324 325 LOCK_CREATE(&sc->bmbx_lock, "Mailbox_lock"); 326 LOCK_CREATE(&sc->dev_lock, "Device_lock"); 327 328 /* initialise the hardware */ 329 rc = oce_hw_init(sc); 330 if (rc) 331 goto pci_res_free; 332 333 oce_read_env_variables(sc); 334 335 oce_get_config(sc); 336 337 setup_max_queues_want(sc); 338 339 rc = oce_setup_intr(sc); 340 if (rc) 341 goto mbox_free; 342 343 rc = oce_queue_init_all(sc); 344 if (rc) 345 goto intr_free; 346 347 rc = oce_attach_ifp(sc); 348 if (rc) 349 goto queues_free; 350 351 #if defined(INET6) || defined(INET) 352 rc = oce_init_lro(sc); 353 if (rc) 354 goto ifp_free; 355 #endif 356 357 rc = oce_hw_start(sc); 358 if (rc) 359 goto lro_free; 360 361 sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 362 oce_add_vlan, sc, EVENTHANDLER_PRI_FIRST); 363 sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 364 oce_del_vlan, sc, EVENTHANDLER_PRI_FIRST); 365 366 rc = oce_stats_init(sc); 367 if (rc) 368 goto vlan_free; 369 370 oce_add_sysctls(sc); 371 372 callout_init(&sc->timer, CALLOUT_MPSAFE); 373 rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc); 374 if (rc) 375 goto stats_free; 376 377 sc->next =NULL; 378 if (softc_tail != NULL) { 379 softc_tail->next = sc; 380 } else { 381 softc_head = sc; 382 } 383 softc_tail = sc; 384 385 return 0; 386 387 stats_free: 388 callout_drain(&sc->timer); 389 oce_stats_free(sc); 390 vlan_free: 391 if (sc->vlan_attach) 392 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); 393 if (sc->vlan_detach) 394 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); 395 oce_hw_intr_disable(sc); 396 lro_free: 397 #if defined(INET6) || defined(INET) 398 oce_free_lro(sc); 399 ifp_free: 400 #endif 401 ether_ifdetach(sc->ifp); 402 if_free(sc->ifp); 403 queues_free: 404 oce_queue_release_all(sc); 405 intr_free: 406 oce_intr_free(sc); 407 mbox_free: 408 oce_dma_free(sc, &sc->bsmbx); 409 pci_res_free: 410 oce_hw_pci_free(sc); 411 LOCK_DESTROY(&sc->dev_lock); 412 LOCK_DESTROY(&sc->bmbx_lock); 413 return rc; 414 415 } 416 417 418 static int 419 oce_detach(device_t dev) 420 { 421 POCE_SOFTC sc = device_get_softc(dev); 422 POCE_SOFTC poce_sc_tmp, *ppoce_sc_tmp1, poce_sc_tmp2 = NULL; 423 424 poce_sc_tmp = softc_head; 425 ppoce_sc_tmp1 = &softc_head; 426 while (poce_sc_tmp != NULL) { 427 if (poce_sc_tmp == sc) { 428 *ppoce_sc_tmp1 = sc->next; 429 if (sc->next == NULL) { 430 softc_tail = poce_sc_tmp2; 431 } 432 break; 433 } 434 poce_sc_tmp2 = poce_sc_tmp; 435 ppoce_sc_tmp1 = &poce_sc_tmp->next; 436 poce_sc_tmp = poce_sc_tmp->next; 437 } 438 439 LOCK(&sc->dev_lock); 440 oce_if_deactivate(sc); 441 UNLOCK(&sc->dev_lock); 442 443 callout_drain(&sc->timer); 444 445 if (sc->vlan_attach != NULL) 446 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); 447 if (sc->vlan_detach != NULL) 448 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); 449 450 ether_ifdetach(sc->ifp); 451 452 if_free(sc->ifp); 453 454 oce_hw_shutdown(sc); 455 456 bus_generic_detach(dev); 457 458 return 0; 459 } 460 461 462 static int 463 oce_shutdown(device_t dev) 464 { 465 int rc; 466 467 rc = oce_detach(dev); 468 469 return rc; 470 } 471 472 473 static int 474 oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 475 { 476 struct ifreq *ifr = (struct ifreq *)data; 477 POCE_SOFTC sc = ifp->if_softc; 478 struct ifi2creq i2c; 479 uint8_t offset = 0; 480 int rc = 0; 481 uint32_t u; 482 483 switch (command) { 484 485 case SIOCGIFMEDIA: 486 rc = ifmedia_ioctl(ifp, ifr, &sc->media, command); 487 break; 488 489 case SIOCSIFMTU: 490 if (ifr->ifr_mtu > OCE_MAX_MTU) 491 rc = EINVAL; 492 else 493 ifp->if_mtu = ifr->ifr_mtu; 494 break; 495 496 case SIOCSIFFLAGS: 497 if (ifp->if_flags & IFF_UP) { 498 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 499 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; 500 oce_init(sc); 501 } 502 device_printf(sc->dev, "Interface Up\n"); 503 } else { 504 LOCK(&sc->dev_lock); 505 506 sc->ifp->if_drv_flags &= 507 ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 508 oce_if_deactivate(sc); 509 510 UNLOCK(&sc->dev_lock); 511 512 device_printf(sc->dev, "Interface Down\n"); 513 } 514 515 if ((ifp->if_flags & IFF_PROMISC) && !sc->promisc) { 516 if (!oce_rxf_set_promiscuous(sc, (1 | (1 << 1)))) 517 sc->promisc = TRUE; 518 } else if (!(ifp->if_flags & IFF_PROMISC) && sc->promisc) { 519 if (!oce_rxf_set_promiscuous(sc, 0)) 520 sc->promisc = FALSE; 521 } 522 523 break; 524 525 case SIOCADDMULTI: 526 case SIOCDELMULTI: 527 rc = oce_hw_update_multicast(sc); 528 if (rc) 529 device_printf(sc->dev, 530 "Update multicast address failed\n"); 531 break; 532 533 case SIOCSIFCAP: 534 u = ifr->ifr_reqcap ^ ifp->if_capenable; 535 536 if (u & IFCAP_TXCSUM) { 537 ifp->if_capenable ^= IFCAP_TXCSUM; 538 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 539 540 if (IFCAP_TSO & ifp->if_capenable && 541 !(IFCAP_TXCSUM & ifp->if_capenable)) { 542 ifp->if_capenable &= ~IFCAP_TSO; 543 ifp->if_hwassist &= ~CSUM_TSO; 544 if_printf(ifp, 545 "TSO disabled due to -txcsum.\n"); 546 } 547 } 548 549 if (u & IFCAP_RXCSUM) 550 ifp->if_capenable ^= IFCAP_RXCSUM; 551 552 if (u & IFCAP_TSO4) { 553 ifp->if_capenable ^= IFCAP_TSO4; 554 555 if (IFCAP_TSO & ifp->if_capenable) { 556 if (IFCAP_TXCSUM & ifp->if_capenable) 557 ifp->if_hwassist |= CSUM_TSO; 558 else { 559 ifp->if_capenable &= ~IFCAP_TSO; 560 ifp->if_hwassist &= ~CSUM_TSO; 561 if_printf(ifp, 562 "Enable txcsum first.\n"); 563 rc = EAGAIN; 564 } 565 } else 566 ifp->if_hwassist &= ~CSUM_TSO; 567 } 568 569 if (u & IFCAP_VLAN_HWTAGGING) 570 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 571 572 if (u & IFCAP_VLAN_HWFILTER) { 573 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 574 oce_vid_config(sc); 575 } 576 #if defined(INET6) || defined(INET) 577 if (u & IFCAP_LRO) { 578 ifp->if_capenable ^= IFCAP_LRO; 579 if(sc->enable_hwlro) { 580 if(ifp->if_capenable & IFCAP_LRO) { 581 rc = oce_mbox_nic_set_iface_lro_config(sc, 1); 582 }else { 583 rc = oce_mbox_nic_set_iface_lro_config(sc, 0); 584 } 585 } 586 } 587 #endif 588 589 break; 590 591 case SIOCGI2C: 592 rc = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c)); 593 if (rc) 594 break; 595 596 if (i2c.dev_addr != PAGE_NUM_A0 && 597 i2c.dev_addr != PAGE_NUM_A2) { 598 rc = EINVAL; 599 break; 600 } 601 602 if (i2c.len > sizeof(i2c.data)) { 603 rc = EINVAL; 604 break; 605 } 606 607 rc = oce_mbox_read_transrecv_data(sc, i2c.dev_addr); 608 if(rc) { 609 rc = -rc; 610 break; 611 } 612 613 if (i2c.dev_addr == PAGE_NUM_A0) 614 offset = i2c.offset; 615 else 616 offset = TRANSCEIVER_A0_SIZE + i2c.offset; 617 618 memcpy(&i2c.data[0], &sfp_vpd_dump_buffer[offset], i2c.len); 619 620 rc = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c)); 621 break; 622 623 case SIOCGPRIVATE_0: 624 rc = oce_handle_passthrough(ifp, data); 625 break; 626 default: 627 rc = ether_ioctl(ifp, command, data); 628 break; 629 } 630 631 return rc; 632 } 633 634 635 static void 636 oce_init(void *arg) 637 { 638 POCE_SOFTC sc = arg; 639 640 LOCK(&sc->dev_lock); 641 642 if (sc->ifp->if_flags & IFF_UP) { 643 oce_if_deactivate(sc); 644 oce_if_activate(sc); 645 } 646 647 UNLOCK(&sc->dev_lock); 648 649 } 650 651 652 static int 653 oce_multiq_start(struct ifnet *ifp, struct mbuf *m) 654 { 655 POCE_SOFTC sc = ifp->if_softc; 656 struct oce_wq *wq = NULL; 657 int queue_index = 0; 658 int status = 0; 659 660 if (!sc->link_status) 661 return ENXIO; 662 663 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 664 queue_index = m->m_pkthdr.flowid % sc->nwqs; 665 666 wq = sc->wq[queue_index]; 667 668 LOCK(&wq->tx_lock); 669 status = oce_multiq_transmit(ifp, m, wq); 670 UNLOCK(&wq->tx_lock); 671 672 return status; 673 674 } 675 676 677 static void 678 oce_multiq_flush(struct ifnet *ifp) 679 { 680 POCE_SOFTC sc = ifp->if_softc; 681 struct mbuf *m; 682 int i = 0; 683 684 for (i = 0; i < sc->nwqs; i++) { 685 while ((m = buf_ring_dequeue_sc(sc->wq[i]->br)) != NULL) 686 m_freem(m); 687 } 688 if_qflush(ifp); 689 } 690 691 692 693 /***************************************************************************** 694 * Driver interrupt routines functions * 695 *****************************************************************************/ 696 697 static void 698 oce_intr(void *arg, int pending) 699 { 700 701 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg; 702 POCE_SOFTC sc = ii->sc; 703 struct oce_eq *eq = ii->eq; 704 struct oce_eqe *eqe; 705 struct oce_cq *cq = NULL; 706 int i, num_eqes = 0; 707 708 709 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, 710 BUS_DMASYNC_POSTWRITE); 711 do { 712 eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe); 713 if (eqe->evnt == 0) 714 break; 715 eqe->evnt = 0; 716 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, 717 BUS_DMASYNC_POSTWRITE); 718 RING_GET(eq->ring, 1); 719 num_eqes++; 720 721 } while (TRUE); 722 723 if (!num_eqes) 724 goto eq_arm; /* Spurious */ 725 726 /* Clear EQ entries, but dont arm */ 727 oce_arm_eq(sc, eq->eq_id, num_eqes, FALSE, FALSE); 728 729 /* Process TX, RX and MCC. But dont arm CQ*/ 730 for (i = 0; i < eq->cq_valid; i++) { 731 cq = eq->cq[i]; 732 (*cq->cq_handler)(cq->cb_arg); 733 } 734 735 /* Arm all cqs connected to this EQ */ 736 for (i = 0; i < eq->cq_valid; i++) { 737 cq = eq->cq[i]; 738 oce_arm_cq(sc, cq->cq_id, 0, TRUE); 739 } 740 741 eq_arm: 742 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); 743 744 return; 745 } 746 747 748 static int 749 oce_setup_intr(POCE_SOFTC sc) 750 { 751 int rc = 0, use_intx = 0; 752 int vector = 0, req_vectors = 0; 753 int tot_req_vectors, tot_vectors; 754 755 if (is_rss_enabled(sc)) 756 req_vectors = MAX((sc->nrqs - 1), sc->nwqs); 757 else 758 req_vectors = 1; 759 760 tot_req_vectors = req_vectors; 761 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 762 if (req_vectors > 1) { 763 tot_req_vectors += OCE_RDMA_VECTORS; 764 sc->roce_intr_count = OCE_RDMA_VECTORS; 765 } 766 } 767 768 if (sc->flags & OCE_FLAGS_MSIX_CAPABLE) { 769 sc->intr_count = req_vectors; 770 tot_vectors = tot_req_vectors; 771 rc = pci_alloc_msix(sc->dev, &tot_vectors); 772 if (rc != 0) { 773 use_intx = 1; 774 pci_release_msi(sc->dev); 775 } else { 776 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 777 if (tot_vectors < tot_req_vectors) { 778 if (sc->intr_count < (2 * OCE_RDMA_VECTORS)) { 779 sc->roce_intr_count = (tot_vectors / 2); 780 } 781 sc->intr_count = tot_vectors - sc->roce_intr_count; 782 } 783 } else { 784 sc->intr_count = tot_vectors; 785 } 786 sc->flags |= OCE_FLAGS_USING_MSIX; 787 } 788 } else 789 use_intx = 1; 790 791 if (use_intx) 792 sc->intr_count = 1; 793 794 /* Scale number of queues based on intr we got */ 795 update_queues_got(sc); 796 797 if (use_intx) { 798 device_printf(sc->dev, "Using legacy interrupt\n"); 799 rc = oce_alloc_intr(sc, vector, oce_intr); 800 if (rc) 801 goto error; 802 } else { 803 for (; vector < sc->intr_count; vector++) { 804 rc = oce_alloc_intr(sc, vector, oce_intr); 805 if (rc) 806 goto error; 807 } 808 } 809 810 return 0; 811 error: 812 oce_intr_free(sc); 813 return rc; 814 } 815 816 817 static int 818 oce_fast_isr(void *arg) 819 { 820 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg; 821 POCE_SOFTC sc = ii->sc; 822 823 if (ii->eq == NULL) 824 return FILTER_STRAY; 825 826 oce_arm_eq(sc, ii->eq->eq_id, 0, FALSE, TRUE); 827 828 taskqueue_enqueue(ii->tq, &ii->task); 829 830 ii->eq->intr++; 831 832 return FILTER_HANDLED; 833 } 834 835 836 static int 837 oce_alloc_intr(POCE_SOFTC sc, int vector, void (*isr) (void *arg, int pending)) 838 { 839 POCE_INTR_INFO ii = &sc->intrs[vector]; 840 int rc = 0, rr; 841 842 if (vector >= OCE_MAX_EQ) 843 return (EINVAL); 844 845 /* Set the resource id for the interrupt. 846 * MSIx is vector + 1 for the resource id, 847 * INTx is 0 for the resource id. 848 */ 849 if (sc->flags & OCE_FLAGS_USING_MSIX) 850 rr = vector + 1; 851 else 852 rr = 0; 853 ii->intr_res = bus_alloc_resource_any(sc->dev, 854 SYS_RES_IRQ, 855 &rr, RF_ACTIVE|RF_SHAREABLE); 856 ii->irq_rr = rr; 857 if (ii->intr_res == NULL) { 858 device_printf(sc->dev, 859 "Could not allocate interrupt\n"); 860 rc = ENXIO; 861 return rc; 862 } 863 864 TASK_INIT(&ii->task, 0, isr, ii); 865 ii->vector = vector; 866 sprintf(ii->task_name, "oce_task[%d]", ii->vector); 867 ii->tq = taskqueue_create_fast(ii->task_name, 868 M_NOWAIT, 869 taskqueue_thread_enqueue, 870 &ii->tq); 871 taskqueue_start_threads(&ii->tq, 1, PI_NET, "%s taskq", 872 device_get_nameunit(sc->dev)); 873 874 ii->sc = sc; 875 rc = bus_setup_intr(sc->dev, 876 ii->intr_res, 877 INTR_TYPE_NET, 878 oce_fast_isr, NULL, ii, &ii->tag); 879 return rc; 880 881 } 882 883 884 void 885 oce_intr_free(POCE_SOFTC sc) 886 { 887 int i = 0; 888 889 for (i = 0; i < sc->intr_count; i++) { 890 891 if (sc->intrs[i].tag != NULL) 892 bus_teardown_intr(sc->dev, sc->intrs[i].intr_res, 893 sc->intrs[i].tag); 894 if (sc->intrs[i].tq != NULL) 895 taskqueue_free(sc->intrs[i].tq); 896 897 if (sc->intrs[i].intr_res != NULL) 898 bus_release_resource(sc->dev, SYS_RES_IRQ, 899 sc->intrs[i].irq_rr, 900 sc->intrs[i].intr_res); 901 sc->intrs[i].tag = NULL; 902 sc->intrs[i].intr_res = NULL; 903 } 904 905 if (sc->flags & OCE_FLAGS_USING_MSIX) 906 pci_release_msi(sc->dev); 907 908 } 909 910 911 912 /****************************************************************************** 913 * Media callbacks functions * 914 ******************************************************************************/ 915 916 static void 917 oce_media_status(struct ifnet *ifp, struct ifmediareq *req) 918 { 919 POCE_SOFTC sc = (POCE_SOFTC) ifp->if_softc; 920 921 922 req->ifm_status = IFM_AVALID; 923 req->ifm_active = IFM_ETHER; 924 925 if (sc->link_status == 1) 926 req->ifm_status |= IFM_ACTIVE; 927 else 928 return; 929 930 switch (sc->link_speed) { 931 case 1: /* 10 Mbps */ 932 req->ifm_active |= IFM_10_T | IFM_FDX; 933 sc->speed = 10; 934 break; 935 case 2: /* 100 Mbps */ 936 req->ifm_active |= IFM_100_TX | IFM_FDX; 937 sc->speed = 100; 938 break; 939 case 3: /* 1 Gbps */ 940 req->ifm_active |= IFM_1000_T | IFM_FDX; 941 sc->speed = 1000; 942 break; 943 case 4: /* 10 Gbps */ 944 req->ifm_active |= IFM_10G_SR | IFM_FDX; 945 sc->speed = 10000; 946 break; 947 case 5: /* 20 Gbps */ 948 req->ifm_active |= IFM_10G_SR | IFM_FDX; 949 sc->speed = 20000; 950 break; 951 case 6: /* 25 Gbps */ 952 req->ifm_active |= IFM_10G_SR | IFM_FDX; 953 sc->speed = 25000; 954 break; 955 case 7: /* 40 Gbps */ 956 req->ifm_active |= IFM_40G_SR4 | IFM_FDX; 957 sc->speed = 40000; 958 break; 959 default: 960 sc->speed = 0; 961 break; 962 } 963 964 return; 965 } 966 967 968 int 969 oce_media_change(struct ifnet *ifp) 970 { 971 return 0; 972 } 973 974 975 static void oce_is_pkt_dest_bmc(POCE_SOFTC sc, 976 struct mbuf *m, boolean_t *os2bmc, 977 struct mbuf **m_new) 978 { 979 struct ether_header *eh = NULL; 980 981 eh = mtod(m, struct ether_header *); 982 983 if (!is_os2bmc_enabled(sc) || *os2bmc) { 984 *os2bmc = FALSE; 985 goto done; 986 } 987 if (!ETHER_IS_MULTICAST(eh->ether_dhost)) 988 goto done; 989 990 if (is_mc_allowed_on_bmc(sc, eh) || 991 is_bc_allowed_on_bmc(sc, eh) || 992 is_arp_allowed_on_bmc(sc, ntohs(eh->ether_type))) { 993 *os2bmc = TRUE; 994 goto done; 995 } 996 997 if (mtod(m, struct ip *)->ip_p == IPPROTO_IPV6) { 998 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); 999 uint8_t nexthdr = ip6->ip6_nxt; 1000 if (nexthdr == IPPROTO_ICMPV6) { 1001 struct icmp6_hdr *icmp6 = (struct icmp6_hdr *)(ip6 + 1); 1002 switch (icmp6->icmp6_type) { 1003 case ND_ROUTER_ADVERT: 1004 *os2bmc = is_ipv6_ra_filt_enabled(sc); 1005 goto done; 1006 case ND_NEIGHBOR_ADVERT: 1007 *os2bmc = is_ipv6_na_filt_enabled(sc); 1008 goto done; 1009 default: 1010 break; 1011 } 1012 } 1013 } 1014 1015 if (mtod(m, struct ip *)->ip_p == IPPROTO_UDP) { 1016 struct ip *ip = mtod(m, struct ip *); 1017 int iphlen = ip->ip_hl << 2; 1018 struct udphdr *uh = (struct udphdr *)((caddr_t)ip + iphlen); 1019 switch (uh->uh_dport) { 1020 case DHCP_CLIENT_PORT: 1021 *os2bmc = is_dhcp_client_filt_enabled(sc); 1022 goto done; 1023 case DHCP_SERVER_PORT: 1024 *os2bmc = is_dhcp_srvr_filt_enabled(sc); 1025 goto done; 1026 case NET_BIOS_PORT1: 1027 case NET_BIOS_PORT2: 1028 *os2bmc = is_nbios_filt_enabled(sc); 1029 goto done; 1030 case DHCPV6_RAS_PORT: 1031 *os2bmc = is_ipv6_ras_filt_enabled(sc); 1032 goto done; 1033 default: 1034 break; 1035 } 1036 } 1037 done: 1038 if (*os2bmc) { 1039 *m_new = m_dup(m, M_NOWAIT); 1040 if (!*m_new) { 1041 *os2bmc = FALSE; 1042 return; 1043 } 1044 *m_new = oce_insert_vlan_tag(sc, *m_new, NULL); 1045 } 1046 } 1047 1048 1049 1050 /***************************************************************************** 1051 * Transmit routines functions * 1052 *****************************************************************************/ 1053 1054 static int 1055 oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index) 1056 { 1057 int rc = 0, i, retry_cnt = 0; 1058 bus_dma_segment_t segs[OCE_MAX_TX_ELEMENTS]; 1059 struct mbuf *m, *m_temp, *m_new = NULL; 1060 struct oce_wq *wq = sc->wq[wq_index]; 1061 struct oce_packet_desc *pd; 1062 struct oce_nic_hdr_wqe *nichdr; 1063 struct oce_nic_frag_wqe *nicfrag; 1064 struct ether_header *eh = NULL; 1065 int num_wqes; 1066 uint32_t reg_value; 1067 boolean_t complete = TRUE; 1068 boolean_t os2bmc = FALSE; 1069 1070 m = *mpp; 1071 if (!m) 1072 return EINVAL; 1073 1074 if (!(m->m_flags & M_PKTHDR)) { 1075 rc = ENXIO; 1076 goto free_ret; 1077 } 1078 1079 /* Don't allow non-TSO packets longer than MTU */ 1080 if (!is_tso_pkt(m)) { 1081 eh = mtod(m, struct ether_header *); 1082 if(m->m_pkthdr.len > ETHER_MAX_FRAME(sc->ifp, eh->ether_type, FALSE)) 1083 goto free_ret; 1084 } 1085 1086 if(oce_tx_asic_stall_verify(sc, m)) { 1087 m = oce_insert_vlan_tag(sc, m, &complete); 1088 if(!m) { 1089 device_printf(sc->dev, "Insertion unsuccessful\n"); 1090 return 0; 1091 } 1092 1093 } 1094 1095 /* Lancer, SH ASIC has a bug wherein Packets that are 32 bytes or less 1096 * may cause a transmit stall on that port. So the work-around is to 1097 * pad short packets (<= 32 bytes) to a 36-byte length. 1098 */ 1099 if(IS_SH(sc) || IS_XE201(sc) ) { 1100 if(m->m_pkthdr.len <= 32) { 1101 char buf[36]; 1102 bzero((void *)buf, 36); 1103 m_append(m, (36 - m->m_pkthdr.len), buf); 1104 } 1105 } 1106 1107 tx_start: 1108 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1109 /* consolidate packet buffers for TSO/LSO segment offload */ 1110 #if defined(INET6) || defined(INET) 1111 m = oce_tso_setup(sc, mpp); 1112 #else 1113 m = NULL; 1114 #endif 1115 if (m == NULL) { 1116 rc = ENXIO; 1117 goto free_ret; 1118 } 1119 } 1120 1121 1122 pd = &wq->pckts[wq->pkt_desc_head]; 1123 1124 retry: 1125 rc = bus_dmamap_load_mbuf_sg(wq->tag, 1126 pd->map, 1127 m, segs, &pd->nsegs, BUS_DMA_NOWAIT); 1128 if (rc == 0) { 1129 num_wqes = pd->nsegs + 1; 1130 if (IS_BE(sc) || IS_SH(sc)) { 1131 /*Dummy required only for BE3.*/ 1132 if (num_wqes & 1) 1133 num_wqes++; 1134 } 1135 if (num_wqes >= RING_NUM_FREE(wq->ring)) { 1136 bus_dmamap_unload(wq->tag, pd->map); 1137 return EBUSY; 1138 } 1139 atomic_store_rel_int(&wq->pkt_desc_head, 1140 (wq->pkt_desc_head + 1) % \ 1141 OCE_WQ_PACKET_ARRAY_SIZE); 1142 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_PREWRITE); 1143 pd->mbuf = m; 1144 1145 nichdr = 1146 RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_hdr_wqe); 1147 nichdr->u0.dw[0] = 0; 1148 nichdr->u0.dw[1] = 0; 1149 nichdr->u0.dw[2] = 0; 1150 nichdr->u0.dw[3] = 0; 1151 1152 nichdr->u0.s.complete = complete; 1153 nichdr->u0.s.mgmt = os2bmc; 1154 nichdr->u0.s.event = 1; 1155 nichdr->u0.s.crc = 1; 1156 nichdr->u0.s.forward = 0; 1157 nichdr->u0.s.ipcs = (m->m_pkthdr.csum_flags & CSUM_IP) ? 1 : 0; 1158 nichdr->u0.s.udpcs = 1159 (m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0; 1160 nichdr->u0.s.tcpcs = 1161 (m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0; 1162 nichdr->u0.s.num_wqe = num_wqes; 1163 nichdr->u0.s.total_length = m->m_pkthdr.len; 1164 1165 if (m->m_flags & M_VLANTAG) { 1166 nichdr->u0.s.vlan = 1; /*Vlan present*/ 1167 nichdr->u0.s.vlan_tag = m->m_pkthdr.ether_vtag; 1168 } 1169 1170 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1171 if (m->m_pkthdr.tso_segsz) { 1172 nichdr->u0.s.lso = 1; 1173 nichdr->u0.s.lso_mss = m->m_pkthdr.tso_segsz; 1174 } 1175 if (!IS_BE(sc) || !IS_SH(sc)) 1176 nichdr->u0.s.ipcs = 1; 1177 } 1178 1179 RING_PUT(wq->ring, 1); 1180 atomic_add_int(&wq->ring->num_used, 1); 1181 1182 for (i = 0; i < pd->nsegs; i++) { 1183 nicfrag = 1184 RING_GET_PRODUCER_ITEM_VA(wq->ring, 1185 struct oce_nic_frag_wqe); 1186 nicfrag->u0.s.rsvd0 = 0; 1187 nicfrag->u0.s.frag_pa_hi = ADDR_HI(segs[i].ds_addr); 1188 nicfrag->u0.s.frag_pa_lo = ADDR_LO(segs[i].ds_addr); 1189 nicfrag->u0.s.frag_len = segs[i].ds_len; 1190 pd->wqe_idx = wq->ring->pidx; 1191 RING_PUT(wq->ring, 1); 1192 atomic_add_int(&wq->ring->num_used, 1); 1193 } 1194 if (num_wqes > (pd->nsegs + 1)) { 1195 nicfrag = 1196 RING_GET_PRODUCER_ITEM_VA(wq->ring, 1197 struct oce_nic_frag_wqe); 1198 nicfrag->u0.dw[0] = 0; 1199 nicfrag->u0.dw[1] = 0; 1200 nicfrag->u0.dw[2] = 0; 1201 nicfrag->u0.dw[3] = 0; 1202 pd->wqe_idx = wq->ring->pidx; 1203 RING_PUT(wq->ring, 1); 1204 atomic_add_int(&wq->ring->num_used, 1); 1205 pd->nsegs++; 1206 } 1207 1208 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1); 1209 wq->tx_stats.tx_reqs++; 1210 wq->tx_stats.tx_wrbs += num_wqes; 1211 wq->tx_stats.tx_bytes += m->m_pkthdr.len; 1212 wq->tx_stats.tx_pkts++; 1213 1214 bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map, 1215 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1216 reg_value = (num_wqes << 16) | wq->wq_id; 1217 1218 /* if os2bmc is not enabled or if the pkt is already tagged as 1219 bmc, do nothing 1220 */ 1221 oce_is_pkt_dest_bmc(sc, m, &os2bmc, &m_new); 1222 1223 OCE_WRITE_REG32(sc, db, wq->db_offset, reg_value); 1224 1225 } else if (rc == EFBIG) { 1226 if (retry_cnt == 0) { 1227 m_temp = m_defrag(m, M_NOWAIT); 1228 if (m_temp == NULL) 1229 goto free_ret; 1230 m = m_temp; 1231 *mpp = m_temp; 1232 retry_cnt = retry_cnt + 1; 1233 goto retry; 1234 } else 1235 goto free_ret; 1236 } else if (rc == ENOMEM) 1237 return rc; 1238 else 1239 goto free_ret; 1240 1241 if (os2bmc) { 1242 m = m_new; 1243 goto tx_start; 1244 } 1245 1246 return 0; 1247 1248 free_ret: 1249 m_freem(*mpp); 1250 *mpp = NULL; 1251 return rc; 1252 } 1253 1254 1255 static void 1256 oce_process_tx_completion(struct oce_wq *wq) 1257 { 1258 struct oce_packet_desc *pd; 1259 POCE_SOFTC sc = (POCE_SOFTC) wq->parent; 1260 struct mbuf *m; 1261 1262 pd = &wq->pckts[wq->pkt_desc_tail]; 1263 atomic_store_rel_int(&wq->pkt_desc_tail, 1264 (wq->pkt_desc_tail + 1) % OCE_WQ_PACKET_ARRAY_SIZE); 1265 atomic_subtract_int(&wq->ring->num_used, pd->nsegs + 1); 1266 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1267 bus_dmamap_unload(wq->tag, pd->map); 1268 1269 m = pd->mbuf; 1270 m_freem(m); 1271 pd->mbuf = NULL; 1272 1273 1274 if (sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) { 1275 if (wq->ring->num_used < (wq->ring->num_items / 2)) { 1276 sc->ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE); 1277 oce_tx_restart(sc, wq); 1278 } 1279 } 1280 } 1281 1282 1283 static void 1284 oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq) 1285 { 1286 1287 if ((sc->ifp->if_drv_flags & IFF_DRV_RUNNING) != IFF_DRV_RUNNING) 1288 return; 1289 1290 #if __FreeBSD_version >= 800000 1291 if (!drbr_empty(sc->ifp, wq->br)) 1292 #else 1293 if (!IFQ_DRV_IS_EMPTY(&sc->ifp->if_snd)) 1294 #endif 1295 taskqueue_enqueue(taskqueue_swi, &wq->txtask); 1296 1297 } 1298 1299 1300 #if defined(INET6) || defined(INET) 1301 static struct mbuf * 1302 oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp) 1303 { 1304 struct mbuf *m; 1305 #ifdef INET 1306 struct ip *ip; 1307 #endif 1308 #ifdef INET6 1309 struct ip6_hdr *ip6; 1310 #endif 1311 struct ether_vlan_header *eh; 1312 struct tcphdr *th; 1313 uint16_t etype; 1314 int total_len = 0, ehdrlen = 0; 1315 1316 m = *mpp; 1317 1318 if (M_WRITABLE(m) == 0) { 1319 m = m_dup(*mpp, M_NOWAIT); 1320 if (!m) 1321 return NULL; 1322 m_freem(*mpp); 1323 *mpp = m; 1324 } 1325 1326 eh = mtod(m, struct ether_vlan_header *); 1327 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 1328 etype = ntohs(eh->evl_proto); 1329 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 1330 } else { 1331 etype = ntohs(eh->evl_encap_proto); 1332 ehdrlen = ETHER_HDR_LEN; 1333 } 1334 1335 switch (etype) { 1336 #ifdef INET 1337 case ETHERTYPE_IP: 1338 ip = (struct ip *)(m->m_data + ehdrlen); 1339 if (ip->ip_p != IPPROTO_TCP) 1340 return NULL; 1341 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2)); 1342 1343 total_len = ehdrlen + (ip->ip_hl << 2) + (th->th_off << 2); 1344 break; 1345 #endif 1346 #ifdef INET6 1347 case ETHERTYPE_IPV6: 1348 ip6 = (struct ip6_hdr *)(m->m_data + ehdrlen); 1349 if (ip6->ip6_nxt != IPPROTO_TCP) 1350 return NULL; 1351 th = (struct tcphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr)); 1352 1353 total_len = ehdrlen + sizeof(struct ip6_hdr) + (th->th_off << 2); 1354 break; 1355 #endif 1356 default: 1357 return NULL; 1358 } 1359 1360 m = m_pullup(m, total_len); 1361 if (!m) 1362 return NULL; 1363 *mpp = m; 1364 return m; 1365 1366 } 1367 #endif /* INET6 || INET */ 1368 1369 void 1370 oce_tx_task(void *arg, int npending) 1371 { 1372 struct oce_wq *wq = arg; 1373 POCE_SOFTC sc = wq->parent; 1374 struct ifnet *ifp = sc->ifp; 1375 int rc = 0; 1376 1377 #if __FreeBSD_version >= 800000 1378 LOCK(&wq->tx_lock); 1379 rc = oce_multiq_transmit(ifp, NULL, wq); 1380 if (rc) { 1381 device_printf(sc->dev, 1382 "TX[%d] restart failed\n", wq->queue_index); 1383 } 1384 UNLOCK(&wq->tx_lock); 1385 #else 1386 oce_start(ifp); 1387 #endif 1388 1389 } 1390 1391 1392 void 1393 oce_start(struct ifnet *ifp) 1394 { 1395 POCE_SOFTC sc = ifp->if_softc; 1396 struct mbuf *m; 1397 int rc = 0; 1398 int def_q = 0; /* Defualt tx queue is 0*/ 1399 1400 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 1401 IFF_DRV_RUNNING) 1402 return; 1403 1404 if (!sc->link_status) 1405 return; 1406 1407 do { 1408 IF_DEQUEUE(&sc->ifp->if_snd, m); 1409 if (m == NULL) 1410 break; 1411 1412 LOCK(&sc->wq[def_q]->tx_lock); 1413 rc = oce_tx(sc, &m, def_q); 1414 UNLOCK(&sc->wq[def_q]->tx_lock); 1415 if (rc) { 1416 if (m != NULL) { 1417 sc->wq[def_q]->tx_stats.tx_stops ++; 1418 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1419 IFQ_DRV_PREPEND(&ifp->if_snd, m); 1420 m = NULL; 1421 } 1422 break; 1423 } 1424 if (m != NULL) 1425 ETHER_BPF_MTAP(ifp, m); 1426 1427 } while (TRUE); 1428 1429 return; 1430 } 1431 1432 1433 /* Handle the Completion Queue for transmit */ 1434 uint16_t 1435 oce_wq_handler(void *arg) 1436 { 1437 struct oce_wq *wq = (struct oce_wq *)arg; 1438 POCE_SOFTC sc = wq->parent; 1439 struct oce_cq *cq = wq->cq; 1440 struct oce_nic_tx_cqe *cqe; 1441 int num_cqes = 0; 1442 1443 LOCK(&wq->tx_compl_lock); 1444 bus_dmamap_sync(cq->ring->dma.tag, 1445 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1446 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 1447 while (cqe->u0.dw[3]) { 1448 DW_SWAP((uint32_t *) cqe, sizeof(oce_wq_cqe)); 1449 1450 wq->ring->cidx = cqe->u0.s.wqe_index + 1; 1451 if (wq->ring->cidx >= wq->ring->num_items) 1452 wq->ring->cidx -= wq->ring->num_items; 1453 1454 oce_process_tx_completion(wq); 1455 wq->tx_stats.tx_compl++; 1456 cqe->u0.dw[3] = 0; 1457 RING_GET(cq->ring, 1); 1458 bus_dmamap_sync(cq->ring->dma.tag, 1459 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1460 cqe = 1461 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 1462 num_cqes++; 1463 } 1464 1465 if (num_cqes) 1466 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); 1467 1468 UNLOCK(&wq->tx_compl_lock); 1469 return num_cqes; 1470 } 1471 1472 1473 static int 1474 oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, struct oce_wq *wq) 1475 { 1476 POCE_SOFTC sc = ifp->if_softc; 1477 int status = 0, queue_index = 0; 1478 struct mbuf *next = NULL; 1479 struct buf_ring *br = NULL; 1480 1481 br = wq->br; 1482 queue_index = wq->queue_index; 1483 1484 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 1485 IFF_DRV_RUNNING) { 1486 if (m != NULL) 1487 status = drbr_enqueue(ifp, br, m); 1488 return status; 1489 } 1490 1491 if (m != NULL) { 1492 if ((status = drbr_enqueue(ifp, br, m)) != 0) 1493 return status; 1494 } 1495 while ((next = drbr_peek(ifp, br)) != NULL) { 1496 if (oce_tx(sc, &next, queue_index)) { 1497 if (next == NULL) { 1498 drbr_advance(ifp, br); 1499 } else { 1500 drbr_putback(ifp, br, next); 1501 wq->tx_stats.tx_stops ++; 1502 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1503 } 1504 break; 1505 } 1506 drbr_advance(ifp, br); 1507 if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len); 1508 if (next->m_flags & M_MCAST) 1509 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); 1510 ETHER_BPF_MTAP(ifp, next); 1511 } 1512 1513 return 0; 1514 } 1515 1516 1517 1518 1519 /***************************************************************************** 1520 * Receive routines functions * 1521 *****************************************************************************/ 1522 1523 static void 1524 oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2) 1525 { 1526 uint32_t *p; 1527 struct ether_header *eh = NULL; 1528 struct tcphdr *tcp_hdr = NULL; 1529 struct ip *ip4_hdr = NULL; 1530 struct ip6_hdr *ip6 = NULL; 1531 uint32_t payload_len = 0; 1532 1533 eh = mtod(m, struct ether_header *); 1534 /* correct IP header */ 1535 if(!cqe2->ipv6_frame) { 1536 ip4_hdr = (struct ip *)((char*)eh + sizeof(struct ether_header)); 1537 ip4_hdr->ip_ttl = cqe2->frame_lifespan; 1538 ip4_hdr->ip_len = htons(cqe2->coalesced_size - sizeof(struct ether_header)); 1539 tcp_hdr = (struct tcphdr *)((char*)ip4_hdr + sizeof(struct ip)); 1540 }else { 1541 ip6 = (struct ip6_hdr *)((char*)eh + sizeof(struct ether_header)); 1542 ip6->ip6_ctlun.ip6_un1.ip6_un1_hlim = cqe2->frame_lifespan; 1543 payload_len = cqe2->coalesced_size - sizeof(struct ether_header) 1544 - sizeof(struct ip6_hdr); 1545 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = htons(payload_len); 1546 tcp_hdr = (struct tcphdr *)((char*)ip6 + sizeof(struct ip6_hdr)); 1547 } 1548 1549 /* correct tcp header */ 1550 tcp_hdr->th_ack = htonl(cqe2->tcp_ack_num); 1551 if(cqe2->push) { 1552 tcp_hdr->th_flags |= TH_PUSH; 1553 } 1554 tcp_hdr->th_win = htons(cqe2->tcp_window); 1555 tcp_hdr->th_sum = 0xffff; 1556 if(cqe2->ts_opt) { 1557 p = (uint32_t *)((char*)tcp_hdr + sizeof(struct tcphdr) + 2); 1558 *p = cqe1->tcp_timestamp_val; 1559 *(p+1) = cqe1->tcp_timestamp_ecr; 1560 } 1561 1562 return; 1563 } 1564 1565 static void 1566 oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m) 1567 { 1568 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1569 uint32_t i = 0, frag_len = 0; 1570 uint32_t len = cqe_info->pkt_size; 1571 struct oce_packet_desc *pd; 1572 struct mbuf *tail = NULL; 1573 1574 for (i = 0; i < cqe_info->num_frags; i++) { 1575 if (rq->ring->cidx == rq->ring->pidx) { 1576 device_printf(sc->dev, 1577 "oce_rx_mbuf_chain: Invalid RX completion - Queue is empty\n"); 1578 return; 1579 } 1580 pd = &rq->pckts[rq->ring->cidx]; 1581 1582 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1583 bus_dmamap_unload(rq->tag, pd->map); 1584 RING_GET(rq->ring, 1); 1585 rq->pending--; 1586 1587 frag_len = (len > rq->cfg.frag_size) ? rq->cfg.frag_size : len; 1588 pd->mbuf->m_len = frag_len; 1589 1590 if (tail != NULL) { 1591 /* additional fragments */ 1592 pd->mbuf->m_flags &= ~M_PKTHDR; 1593 tail->m_next = pd->mbuf; 1594 if(rq->islro) 1595 tail->m_nextpkt = NULL; 1596 tail = pd->mbuf; 1597 } else { 1598 /* first fragment, fill out much of the packet header */ 1599 pd->mbuf->m_pkthdr.len = len; 1600 if(rq->islro) 1601 pd->mbuf->m_nextpkt = NULL; 1602 pd->mbuf->m_pkthdr.csum_flags = 0; 1603 if (IF_CSUM_ENABLED(sc)) { 1604 if (cqe_info->l4_cksum_pass) { 1605 if(!cqe_info->ipv6_frame) { /* IPV4 */ 1606 pd->mbuf->m_pkthdr.csum_flags |= 1607 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1608 }else { /* IPV6 frame */ 1609 if(rq->islro) { 1610 pd->mbuf->m_pkthdr.csum_flags |= 1611 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1612 } 1613 } 1614 pd->mbuf->m_pkthdr.csum_data = 0xffff; 1615 } 1616 if (cqe_info->ip_cksum_pass) { 1617 pd->mbuf->m_pkthdr.csum_flags |= 1618 (CSUM_IP_CHECKED|CSUM_IP_VALID); 1619 } 1620 } 1621 *m = tail = pd->mbuf; 1622 } 1623 pd->mbuf = NULL; 1624 len -= frag_len; 1625 } 1626 1627 return; 1628 } 1629 1630 static void 1631 oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2) 1632 { 1633 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1634 struct nic_hwlro_cqe_part1 *cqe1 = NULL; 1635 struct mbuf *m = NULL; 1636 struct oce_common_cqe_info cq_info; 1637 1638 /* parse cqe */ 1639 if(cqe2 == NULL) { 1640 cq_info.pkt_size = cqe->pkt_size; 1641 cq_info.vtag = cqe->vlan_tag; 1642 cq_info.l4_cksum_pass = cqe->l4_cksum_pass; 1643 cq_info.ip_cksum_pass = cqe->ip_cksum_pass; 1644 cq_info.ipv6_frame = cqe->ipv6_frame; 1645 cq_info.vtp = cqe->vtp; 1646 cq_info.qnq = cqe->qnq; 1647 }else { 1648 cqe1 = (struct nic_hwlro_cqe_part1 *)cqe; 1649 cq_info.pkt_size = cqe2->coalesced_size; 1650 cq_info.vtag = cqe2->vlan_tag; 1651 cq_info.l4_cksum_pass = cqe2->l4_cksum_pass; 1652 cq_info.ip_cksum_pass = cqe2->ip_cksum_pass; 1653 cq_info.ipv6_frame = cqe2->ipv6_frame; 1654 cq_info.vtp = cqe2->vtp; 1655 cq_info.qnq = cqe1->qnq; 1656 } 1657 1658 cq_info.vtag = BSWAP_16(cq_info.vtag); 1659 1660 cq_info.num_frags = cq_info.pkt_size / rq->cfg.frag_size; 1661 if(cq_info.pkt_size % rq->cfg.frag_size) 1662 cq_info.num_frags++; 1663 1664 oce_rx_mbuf_chain(rq, &cq_info, &m); 1665 1666 if (m) { 1667 if(cqe2) { 1668 //assert(cqe2->valid != 0); 1669 1670 //assert(cqe2->cqe_type != 2); 1671 oce_correct_header(m, cqe1, cqe2); 1672 } 1673 1674 m->m_pkthdr.rcvif = sc->ifp; 1675 #if __FreeBSD_version >= 800000 1676 if (rq->queue_index) 1677 m->m_pkthdr.flowid = (rq->queue_index - 1); 1678 else 1679 m->m_pkthdr.flowid = rq->queue_index; 1680 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 1681 #endif 1682 /* This deternies if vlan tag is Valid */ 1683 if (cq_info.vtp) { 1684 if (sc->function_mode & FNM_FLEX10_MODE) { 1685 /* FLEX10. If QnQ is not set, neglect VLAN */ 1686 if (cq_info.qnq) { 1687 m->m_pkthdr.ether_vtag = cq_info.vtag; 1688 m->m_flags |= M_VLANTAG; 1689 } 1690 } else if (sc->pvid != (cq_info.vtag & VLAN_VID_MASK)) { 1691 /* In UMC mode generally pvid will be striped by 1692 hw. But in some cases we have seen it comes 1693 with pvid. So if pvid == vlan, neglect vlan. 1694 */ 1695 m->m_pkthdr.ether_vtag = cq_info.vtag; 1696 m->m_flags |= M_VLANTAG; 1697 } 1698 } 1699 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 1700 1701 (*sc->ifp->if_input) (sc->ifp, m); 1702 1703 /* Update rx stats per queue */ 1704 rq->rx_stats.rx_pkts++; 1705 rq->rx_stats.rx_bytes += cq_info.pkt_size; 1706 rq->rx_stats.rx_frags += cq_info.num_frags; 1707 rq->rx_stats.rx_ucast_pkts++; 1708 } 1709 return; 1710 } 1711 1712 static void 1713 oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe) 1714 { 1715 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1716 int len; 1717 struct mbuf *m = NULL; 1718 struct oce_common_cqe_info cq_info; 1719 uint16_t vtag = 0; 1720 1721 /* Is it a flush compl that has no data */ 1722 if(!cqe->u0.s.num_fragments) 1723 goto exit; 1724 1725 len = cqe->u0.s.pkt_size; 1726 if (!len) { 1727 /*partial DMA workaround for Lancer*/ 1728 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); 1729 goto exit; 1730 } 1731 1732 if (!oce_cqe_portid_valid(sc, cqe)) { 1733 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); 1734 goto exit; 1735 } 1736 1737 /* Get vlan_tag value */ 1738 if(IS_BE(sc) || IS_SH(sc)) 1739 vtag = BSWAP_16(cqe->u0.s.vlan_tag); 1740 else 1741 vtag = cqe->u0.s.vlan_tag; 1742 1743 cq_info.l4_cksum_pass = cqe->u0.s.l4_cksum_pass; 1744 cq_info.ip_cksum_pass = cqe->u0.s.ip_cksum_pass; 1745 cq_info.ipv6_frame = cqe->u0.s.ip_ver; 1746 cq_info.num_frags = cqe->u0.s.num_fragments; 1747 cq_info.pkt_size = cqe->u0.s.pkt_size; 1748 1749 oce_rx_mbuf_chain(rq, &cq_info, &m); 1750 1751 if (m) { 1752 m->m_pkthdr.rcvif = sc->ifp; 1753 #if __FreeBSD_version >= 800000 1754 if (rq->queue_index) 1755 m->m_pkthdr.flowid = (rq->queue_index - 1); 1756 else 1757 m->m_pkthdr.flowid = rq->queue_index; 1758 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 1759 #endif 1760 /* This deternies if vlan tag is Valid */ 1761 if (oce_cqe_vtp_valid(sc, cqe)) { 1762 if (sc->function_mode & FNM_FLEX10_MODE) { 1763 /* FLEX10. If QnQ is not set, neglect VLAN */ 1764 if (cqe->u0.s.qnq) { 1765 m->m_pkthdr.ether_vtag = vtag; 1766 m->m_flags |= M_VLANTAG; 1767 } 1768 } else if (sc->pvid != (vtag & VLAN_VID_MASK)) { 1769 /* In UMC mode generally pvid will be striped by 1770 hw. But in some cases we have seen it comes 1771 with pvid. So if pvid == vlan, neglect vlan. 1772 */ 1773 m->m_pkthdr.ether_vtag = vtag; 1774 m->m_flags |= M_VLANTAG; 1775 } 1776 } 1777 1778 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 1779 #if defined(INET6) || defined(INET) 1780 /* Try to queue to LRO */ 1781 if (IF_LRO_ENABLED(sc) && 1782 (cqe->u0.s.ip_cksum_pass) && 1783 (cqe->u0.s.l4_cksum_pass) && 1784 (!cqe->u0.s.ip_ver) && 1785 (rq->lro.lro_cnt != 0)) { 1786 1787 if (tcp_lro_rx(&rq->lro, m, 0) == 0) { 1788 rq->lro_pkts_queued ++; 1789 goto post_done; 1790 } 1791 /* If LRO posting fails then try to post to STACK */ 1792 } 1793 #endif 1794 1795 (*sc->ifp->if_input) (sc->ifp, m); 1796 #if defined(INET6) || defined(INET) 1797 post_done: 1798 #endif 1799 /* Update rx stats per queue */ 1800 rq->rx_stats.rx_pkts++; 1801 rq->rx_stats.rx_bytes += cqe->u0.s.pkt_size; 1802 rq->rx_stats.rx_frags += cqe->u0.s.num_fragments; 1803 if (cqe->u0.s.pkt_type == OCE_MULTICAST_PACKET) 1804 rq->rx_stats.rx_mcast_pkts++; 1805 if (cqe->u0.s.pkt_type == OCE_UNICAST_PACKET) 1806 rq->rx_stats.rx_ucast_pkts++; 1807 } 1808 exit: 1809 return; 1810 } 1811 1812 1813 void 1814 oce_discard_rx_comp(struct oce_rq *rq, int num_frags) 1815 { 1816 uint32_t i = 0; 1817 struct oce_packet_desc *pd; 1818 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1819 1820 for (i = 0; i < num_frags; i++) { 1821 if (rq->ring->cidx == rq->ring->pidx) { 1822 device_printf(sc->dev, 1823 "oce_discard_rx_comp: Invalid RX completion - Queue is empty\n"); 1824 return; 1825 } 1826 pd = &rq->pckts[rq->ring->cidx]; 1827 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1828 bus_dmamap_unload(rq->tag, pd->map); 1829 if (pd->mbuf != NULL) { 1830 m_freem(pd->mbuf); 1831 pd->mbuf = NULL; 1832 } 1833 1834 RING_GET(rq->ring, 1); 1835 rq->pending--; 1836 } 1837 } 1838 1839 1840 static int 1841 oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe) 1842 { 1843 struct oce_nic_rx_cqe_v1 *cqe_v1; 1844 int vtp = 0; 1845 1846 if (sc->be3_native) { 1847 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe; 1848 vtp = cqe_v1->u0.s.vlan_tag_present; 1849 } else 1850 vtp = cqe->u0.s.vlan_tag_present; 1851 1852 return vtp; 1853 1854 } 1855 1856 1857 static int 1858 oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe) 1859 { 1860 struct oce_nic_rx_cqe_v1 *cqe_v1; 1861 int port_id = 0; 1862 1863 if (sc->be3_native && (IS_BE(sc) || IS_SH(sc))) { 1864 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe; 1865 port_id = cqe_v1->u0.s.port; 1866 if (sc->port_id != port_id) 1867 return 0; 1868 } else 1869 ;/* For BE3 legacy and Lancer this is dummy */ 1870 1871 return 1; 1872 1873 } 1874 1875 #if defined(INET6) || defined(INET) 1876 void 1877 oce_rx_flush_lro(struct oce_rq *rq) 1878 { 1879 struct lro_ctrl *lro = &rq->lro; 1880 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1881 1882 if (!IF_LRO_ENABLED(sc)) 1883 return; 1884 1885 tcp_lro_flush_all(lro); 1886 rq->lro_pkts_queued = 0; 1887 1888 return; 1889 } 1890 1891 1892 static int 1893 oce_init_lro(POCE_SOFTC sc) 1894 { 1895 struct lro_ctrl *lro = NULL; 1896 int i = 0, rc = 0; 1897 1898 for (i = 0; i < sc->nrqs; i++) { 1899 lro = &sc->rq[i]->lro; 1900 rc = tcp_lro_init(lro); 1901 if (rc != 0) { 1902 device_printf(sc->dev, "LRO init failed\n"); 1903 return rc; 1904 } 1905 lro->ifp = sc->ifp; 1906 } 1907 1908 return rc; 1909 } 1910 1911 1912 void 1913 oce_free_lro(POCE_SOFTC sc) 1914 { 1915 struct lro_ctrl *lro = NULL; 1916 int i = 0; 1917 1918 for (i = 0; i < sc->nrqs; i++) { 1919 lro = &sc->rq[i]->lro; 1920 if (lro) 1921 tcp_lro_free(lro); 1922 } 1923 } 1924 #endif 1925 1926 int 1927 oce_alloc_rx_bufs(struct oce_rq *rq, int count) 1928 { 1929 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1930 int i, in, rc; 1931 struct oce_packet_desc *pd; 1932 bus_dma_segment_t segs[6]; 1933 int nsegs, added = 0; 1934 struct oce_nic_rqe *rqe; 1935 pd_rxulp_db_t rxdb_reg; 1936 uint32_t val = 0; 1937 uint32_t oce_max_rq_posts = 64; 1938 1939 bzero(&rxdb_reg, sizeof(pd_rxulp_db_t)); 1940 for (i = 0; i < count; i++) { 1941 in = (rq->ring->pidx + 1) % OCE_RQ_PACKET_ARRAY_SIZE; 1942 1943 pd = &rq->pckts[rq->ring->pidx]; 1944 pd->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, oce_rq_buf_size); 1945 if (pd->mbuf == NULL) { 1946 device_printf(sc->dev, "mbuf allocation failed, size = %d\n",oce_rq_buf_size); 1947 break; 1948 } 1949 pd->mbuf->m_nextpkt = NULL; 1950 1951 pd->mbuf->m_len = pd->mbuf->m_pkthdr.len = rq->cfg.frag_size; 1952 1953 rc = bus_dmamap_load_mbuf_sg(rq->tag, 1954 pd->map, 1955 pd->mbuf, 1956 segs, &nsegs, BUS_DMA_NOWAIT); 1957 if (rc) { 1958 m_free(pd->mbuf); 1959 device_printf(sc->dev, "bus_dmamap_load_mbuf_sg failed rc = %d\n", rc); 1960 break; 1961 } 1962 1963 if (nsegs != 1) { 1964 i--; 1965 continue; 1966 } 1967 1968 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_PREREAD); 1969 1970 rqe = RING_GET_PRODUCER_ITEM_VA(rq->ring, struct oce_nic_rqe); 1971 rqe->u0.s.frag_pa_hi = ADDR_HI(segs[0].ds_addr); 1972 rqe->u0.s.frag_pa_lo = ADDR_LO(segs[0].ds_addr); 1973 DW_SWAP(u32ptr(rqe), sizeof(struct oce_nic_rqe)); 1974 RING_PUT(rq->ring, 1); 1975 added++; 1976 rq->pending++; 1977 } 1978 oce_max_rq_posts = sc->enable_hwlro ? OCE_HWLRO_MAX_RQ_POSTS : OCE_MAX_RQ_POSTS; 1979 if (added != 0) { 1980 for (i = added / oce_max_rq_posts; i > 0; i--) { 1981 rxdb_reg.bits.num_posted = oce_max_rq_posts; 1982 rxdb_reg.bits.qid = rq->rq_id; 1983 if(rq->islro) { 1984 val |= rq->rq_id & DB_LRO_RQ_ID_MASK; 1985 val |= oce_max_rq_posts << 16; 1986 OCE_WRITE_REG32(sc, db, DB_OFFSET, val); 1987 }else { 1988 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); 1989 } 1990 added -= oce_max_rq_posts; 1991 } 1992 if (added > 0) { 1993 rxdb_reg.bits.qid = rq->rq_id; 1994 rxdb_reg.bits.num_posted = added; 1995 if(rq->islro) { 1996 val |= rq->rq_id & DB_LRO_RQ_ID_MASK; 1997 val |= added << 16; 1998 OCE_WRITE_REG32(sc, db, DB_OFFSET, val); 1999 }else { 2000 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); 2001 } 2002 } 2003 } 2004 2005 return 0; 2006 } 2007 2008 static void 2009 oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq) 2010 { 2011 if (num_cqes) { 2012 oce_arm_cq(sc, rq->cq->cq_id, num_cqes, FALSE); 2013 if(!sc->enable_hwlro) { 2014 if((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) > 1) 2015 oce_alloc_rx_bufs(rq, ((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) - 1)); 2016 }else { 2017 if ((OCE_RQ_PACKET_ARRAY_SIZE -1 - rq->pending) > 64) 2018 oce_alloc_rx_bufs(rq, 64); 2019 } 2020 } 2021 2022 return; 2023 } 2024 2025 uint16_t 2026 oce_rq_handler_lro(void *arg) 2027 { 2028 struct oce_rq *rq = (struct oce_rq *)arg; 2029 struct oce_cq *cq = rq->cq; 2030 POCE_SOFTC sc = rq->parent; 2031 struct nic_hwlro_singleton_cqe *cqe; 2032 struct nic_hwlro_cqe_part2 *cqe2; 2033 int num_cqes = 0; 2034 2035 LOCK(&rq->rx_lock); 2036 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2037 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); 2038 while (cqe->valid) { 2039 if(cqe->cqe_type == 0) { /* singleton cqe */ 2040 /* we should not get singleton cqe after cqe1 on same rq */ 2041 if(rq->cqe_firstpart != NULL) { 2042 device_printf(sc->dev, "Got singleton cqe after cqe1 \n"); 2043 goto exit_rq_handler_lro; 2044 } 2045 if(cqe->error != 0) { 2046 rq->rx_stats.rxcp_err++; 2047 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2048 } 2049 oce_rx_lro(rq, cqe, NULL); 2050 rq->rx_stats.rx_compl++; 2051 cqe->valid = 0; 2052 RING_GET(cq->ring, 1); 2053 num_cqes++; 2054 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2055 break; 2056 }else if(cqe->cqe_type == 0x1) { /* first part */ 2057 /* we should not get cqe1 after cqe1 on same rq */ 2058 if(rq->cqe_firstpart != NULL) { 2059 device_printf(sc->dev, "Got cqe1 after cqe1 \n"); 2060 goto exit_rq_handler_lro; 2061 } 2062 rq->cqe_firstpart = (struct nic_hwlro_cqe_part1 *)cqe; 2063 RING_GET(cq->ring, 1); 2064 }else if(cqe->cqe_type == 0x2) { /* second part */ 2065 cqe2 = (struct nic_hwlro_cqe_part2 *)cqe; 2066 if(cqe2->error != 0) { 2067 rq->rx_stats.rxcp_err++; 2068 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2069 } 2070 /* We should not get cqe2 without cqe1 */ 2071 if(rq->cqe_firstpart == NULL) { 2072 device_printf(sc->dev, "Got cqe2 without cqe1 \n"); 2073 goto exit_rq_handler_lro; 2074 } 2075 oce_rx_lro(rq, (struct nic_hwlro_singleton_cqe *)rq->cqe_firstpart, cqe2); 2076 2077 rq->rx_stats.rx_compl++; 2078 rq->cqe_firstpart->valid = 0; 2079 cqe2->valid = 0; 2080 rq->cqe_firstpart = NULL; 2081 2082 RING_GET(cq->ring, 1); 2083 num_cqes += 2; 2084 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2085 break; 2086 } 2087 2088 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2089 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); 2090 } 2091 oce_check_rx_bufs(sc, num_cqes, rq); 2092 exit_rq_handler_lro: 2093 UNLOCK(&rq->rx_lock); 2094 return 0; 2095 } 2096 2097 /* Handle the Completion Queue for receive */ 2098 uint16_t 2099 oce_rq_handler(void *arg) 2100 { 2101 struct oce_rq *rq = (struct oce_rq *)arg; 2102 struct oce_cq *cq = rq->cq; 2103 POCE_SOFTC sc = rq->parent; 2104 struct oce_nic_rx_cqe *cqe; 2105 int num_cqes = 0; 2106 2107 if(rq->islro) { 2108 oce_rq_handler_lro(arg); 2109 return 0; 2110 } 2111 LOCK(&rq->rx_lock); 2112 bus_dmamap_sync(cq->ring->dma.tag, 2113 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2114 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); 2115 while (cqe->u0.dw[2]) { 2116 DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe)); 2117 2118 if (cqe->u0.s.error == 0) { 2119 oce_rx(rq, cqe); 2120 } else { 2121 rq->rx_stats.rxcp_err++; 2122 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2123 /* Post L3/L4 errors to stack.*/ 2124 oce_rx(rq, cqe); 2125 } 2126 rq->rx_stats.rx_compl++; 2127 cqe->u0.dw[2] = 0; 2128 2129 #if defined(INET6) || defined(INET) 2130 if (IF_LRO_ENABLED(sc) && rq->lro_pkts_queued >= 16) { 2131 oce_rx_flush_lro(rq); 2132 } 2133 #endif 2134 2135 RING_GET(cq->ring, 1); 2136 bus_dmamap_sync(cq->ring->dma.tag, 2137 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2138 cqe = 2139 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); 2140 num_cqes++; 2141 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2142 break; 2143 } 2144 2145 #if defined(INET6) || defined(INET) 2146 if (IF_LRO_ENABLED(sc)) 2147 oce_rx_flush_lro(rq); 2148 #endif 2149 2150 oce_check_rx_bufs(sc, num_cqes, rq); 2151 UNLOCK(&rq->rx_lock); 2152 return 0; 2153 2154 } 2155 2156 2157 2158 2159 /***************************************************************************** 2160 * Helper function prototypes in this file * 2161 *****************************************************************************/ 2162 2163 static int 2164 oce_attach_ifp(POCE_SOFTC sc) 2165 { 2166 2167 sc->ifp = if_alloc(IFT_ETHER); 2168 if (!sc->ifp) 2169 return ENOMEM; 2170 2171 ifmedia_init(&sc->media, IFM_IMASK, oce_media_change, oce_media_status); 2172 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2173 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 2174 2175 sc->ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST; 2176 sc->ifp->if_ioctl = oce_ioctl; 2177 sc->ifp->if_start = oce_start; 2178 sc->ifp->if_init = oce_init; 2179 sc->ifp->if_mtu = ETHERMTU; 2180 sc->ifp->if_softc = sc; 2181 #if __FreeBSD_version >= 800000 2182 sc->ifp->if_transmit = oce_multiq_start; 2183 sc->ifp->if_qflush = oce_multiq_flush; 2184 #endif 2185 2186 if_initname(sc->ifp, 2187 device_get_name(sc->dev), device_get_unit(sc->dev)); 2188 2189 sc->ifp->if_snd.ifq_drv_maxlen = OCE_MAX_TX_DESC - 1; 2190 IFQ_SET_MAXLEN(&sc->ifp->if_snd, sc->ifp->if_snd.ifq_drv_maxlen); 2191 IFQ_SET_READY(&sc->ifp->if_snd); 2192 2193 sc->ifp->if_hwassist = OCE_IF_HWASSIST; 2194 sc->ifp->if_hwassist |= CSUM_TSO; 2195 sc->ifp->if_hwassist |= (CSUM_IP | CSUM_TCP | CSUM_UDP); 2196 2197 sc->ifp->if_capabilities = OCE_IF_CAPABILITIES; 2198 sc->ifp->if_capabilities |= IFCAP_HWCSUM; 2199 sc->ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 2200 2201 #if defined(INET6) || defined(INET) 2202 sc->ifp->if_capabilities |= IFCAP_TSO; 2203 sc->ifp->if_capabilities |= IFCAP_LRO; 2204 sc->ifp->if_capabilities |= IFCAP_VLAN_HWTSO; 2205 #endif 2206 2207 sc->ifp->if_capenable = sc->ifp->if_capabilities; 2208 sc->ifp->if_baudrate = IF_Gbps(10); 2209 2210 #if __FreeBSD_version >= 1000000 2211 sc->ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 2212 sc->ifp->if_hw_tsomaxsegcount = OCE_MAX_TX_ELEMENTS; 2213 sc->ifp->if_hw_tsomaxsegsize = 4096; 2214 #endif 2215 2216 ether_ifattach(sc->ifp, sc->macaddr.mac_addr); 2217 2218 return 0; 2219 } 2220 2221 2222 static void 2223 oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag) 2224 { 2225 POCE_SOFTC sc = ifp->if_softc; 2226 2227 if (ifp->if_softc != arg) 2228 return; 2229 if ((vtag == 0) || (vtag > 4095)) 2230 return; 2231 2232 sc->vlan_tag[vtag] = 1; 2233 sc->vlans_added++; 2234 if (sc->vlans_added <= (sc->max_vlans + 1)) 2235 oce_vid_config(sc); 2236 } 2237 2238 2239 static void 2240 oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag) 2241 { 2242 POCE_SOFTC sc = ifp->if_softc; 2243 2244 if (ifp->if_softc != arg) 2245 return; 2246 if ((vtag == 0) || (vtag > 4095)) 2247 return; 2248 2249 sc->vlan_tag[vtag] = 0; 2250 sc->vlans_added--; 2251 oce_vid_config(sc); 2252 } 2253 2254 2255 /* 2256 * A max of 64 vlans can be configured in BE. If the user configures 2257 * more, place the card in vlan promiscuous mode. 2258 */ 2259 static int 2260 oce_vid_config(POCE_SOFTC sc) 2261 { 2262 struct normal_vlan vtags[MAX_VLANFILTER_SIZE]; 2263 uint16_t ntags = 0, i; 2264 int status = 0; 2265 2266 if ((sc->vlans_added <= MAX_VLANFILTER_SIZE) && 2267 (sc->ifp->if_capenable & IFCAP_VLAN_HWFILTER)) { 2268 for (i = 0; i < MAX_VLANS; i++) { 2269 if (sc->vlan_tag[i]) { 2270 vtags[ntags].vtag = i; 2271 ntags++; 2272 } 2273 } 2274 if (ntags) 2275 status = oce_config_vlan(sc, (uint8_t) sc->if_id, 2276 vtags, ntags, 1, 0); 2277 } else 2278 status = oce_config_vlan(sc, (uint8_t) sc->if_id, 2279 NULL, 0, 1, 1); 2280 return status; 2281 } 2282 2283 2284 static void 2285 oce_mac_addr_set(POCE_SOFTC sc) 2286 { 2287 uint32_t old_pmac_id = sc->pmac_id; 2288 int status = 0; 2289 2290 2291 status = bcmp((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr, 2292 sc->macaddr.size_of_struct); 2293 if (!status) 2294 return; 2295 2296 status = oce_mbox_macaddr_add(sc, (uint8_t *)(IF_LLADDR(sc->ifp)), 2297 sc->if_id, &sc->pmac_id); 2298 if (!status) { 2299 status = oce_mbox_macaddr_del(sc, sc->if_id, old_pmac_id); 2300 bcopy((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr, 2301 sc->macaddr.size_of_struct); 2302 } 2303 if (status) 2304 device_printf(sc->dev, "Failed update macaddress\n"); 2305 2306 } 2307 2308 2309 static int 2310 oce_handle_passthrough(struct ifnet *ifp, caddr_t data) 2311 { 2312 POCE_SOFTC sc = ifp->if_softc; 2313 struct ifreq *ifr = (struct ifreq *)data; 2314 int rc = ENXIO; 2315 char cookie[32] = {0}; 2316 void *priv_data = ifr_data_get_ptr(ifr); 2317 void *ioctl_ptr; 2318 uint32_t req_size; 2319 struct mbx_hdr req; 2320 OCE_DMA_MEM dma_mem; 2321 struct mbx_common_get_cntl_attr *fw_cmd; 2322 2323 if (copyin(priv_data, cookie, strlen(IOCTL_COOKIE))) 2324 return EFAULT; 2325 2326 if (memcmp(cookie, IOCTL_COOKIE, strlen(IOCTL_COOKIE))) 2327 return EINVAL; 2328 2329 ioctl_ptr = (char *)priv_data + strlen(IOCTL_COOKIE); 2330 if (copyin(ioctl_ptr, &req, sizeof(struct mbx_hdr))) 2331 return EFAULT; 2332 2333 req_size = le32toh(req.u0.req.request_length); 2334 if (req_size > 65536) 2335 return EINVAL; 2336 2337 req_size += sizeof(struct mbx_hdr); 2338 rc = oce_dma_alloc(sc, req_size, &dma_mem, 0); 2339 if (rc) 2340 return ENOMEM; 2341 2342 if (copyin(ioctl_ptr, OCE_DMAPTR(&dma_mem,char), req_size)) { 2343 rc = EFAULT; 2344 goto dma_free; 2345 } 2346 2347 rc = oce_pass_through_mbox(sc, &dma_mem, req_size); 2348 if (rc) { 2349 rc = EIO; 2350 goto dma_free; 2351 } 2352 2353 if (copyout(OCE_DMAPTR(&dma_mem,char), ioctl_ptr, req_size)) 2354 rc = EFAULT; 2355 2356 /* 2357 firmware is filling all the attributes for this ioctl except 2358 the driver version..so fill it 2359 */ 2360 if(req.u0.rsp.opcode == OPCODE_COMMON_GET_CNTL_ATTRIBUTES) { 2361 fw_cmd = (struct mbx_common_get_cntl_attr *) ioctl_ptr; 2362 strncpy(fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str, 2363 COMPONENT_REVISION, strlen(COMPONENT_REVISION)); 2364 } 2365 2366 dma_free: 2367 oce_dma_free(sc, &dma_mem); 2368 return rc; 2369 2370 } 2371 2372 static void 2373 oce_eqd_set_periodic(POCE_SOFTC sc) 2374 { 2375 struct oce_set_eqd set_eqd[OCE_MAX_EQ]; 2376 struct oce_aic_obj *aic; 2377 struct oce_eq *eqo; 2378 uint64_t now = 0, delta; 2379 int eqd, i, num = 0; 2380 uint32_t tx_reqs = 0, rxpkts = 0, pps; 2381 struct oce_wq *wq; 2382 struct oce_rq *rq; 2383 2384 #define ticks_to_msecs(t) (1000 * (t) / hz) 2385 2386 for (i = 0 ; i < sc->neqs; i++) { 2387 eqo = sc->eq[i]; 2388 aic = &sc->aic_obj[i]; 2389 /* When setting the static eq delay from the user space */ 2390 if (!aic->enable) { 2391 if (aic->ticks) 2392 aic->ticks = 0; 2393 eqd = aic->et_eqd; 2394 goto modify_eqd; 2395 } 2396 2397 if (i == 0) { 2398 rq = sc->rq[0]; 2399 rxpkts = rq->rx_stats.rx_pkts; 2400 } else 2401 rxpkts = 0; 2402 if (i + 1 < sc->nrqs) { 2403 rq = sc->rq[i + 1]; 2404 rxpkts += rq->rx_stats.rx_pkts; 2405 } 2406 if (i < sc->nwqs) { 2407 wq = sc->wq[i]; 2408 tx_reqs = wq->tx_stats.tx_reqs; 2409 } else 2410 tx_reqs = 0; 2411 now = ticks; 2412 2413 if (!aic->ticks || now < aic->ticks || 2414 rxpkts < aic->prev_rxpkts || tx_reqs < aic->prev_txreqs) { 2415 aic->prev_rxpkts = rxpkts; 2416 aic->prev_txreqs = tx_reqs; 2417 aic->ticks = now; 2418 continue; 2419 } 2420 2421 delta = ticks_to_msecs(now - aic->ticks); 2422 2423 pps = (((uint32_t)(rxpkts - aic->prev_rxpkts) * 1000) / delta) + 2424 (((uint32_t)(tx_reqs - aic->prev_txreqs) * 1000) / delta); 2425 eqd = (pps / 15000) << 2; 2426 if (eqd < 8) 2427 eqd = 0; 2428 2429 /* Make sure that the eq delay is in the known range */ 2430 eqd = min(eqd, aic->max_eqd); 2431 eqd = max(eqd, aic->min_eqd); 2432 2433 aic->prev_rxpkts = rxpkts; 2434 aic->prev_txreqs = tx_reqs; 2435 aic->ticks = now; 2436 2437 modify_eqd: 2438 if (eqd != aic->cur_eqd) { 2439 set_eqd[num].delay_multiplier = (eqd * 65)/100; 2440 set_eqd[num].eq_id = eqo->eq_id; 2441 aic->cur_eqd = eqd; 2442 num++; 2443 } 2444 } 2445 2446 /* Is there atleast one eq that needs to be modified? */ 2447 for(i = 0; i < num; i += 8) { 2448 if((num - i) >=8 ) 2449 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], 8); 2450 else 2451 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], (num - i)); 2452 } 2453 2454 } 2455 2456 static void oce_detect_hw_error(POCE_SOFTC sc) 2457 { 2458 2459 uint32_t ue_low = 0, ue_high = 0, ue_low_mask = 0, ue_high_mask = 0; 2460 uint32_t sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0; 2461 uint32_t i; 2462 2463 if (sc->hw_error) 2464 return; 2465 2466 if (IS_XE201(sc)) { 2467 sliport_status = OCE_READ_REG32(sc, db, SLIPORT_STATUS_OFFSET); 2468 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2469 sliport_err1 = OCE_READ_REG32(sc, db, SLIPORT_ERROR1_OFFSET); 2470 sliport_err2 = OCE_READ_REG32(sc, db, SLIPORT_ERROR2_OFFSET); 2471 } 2472 } else { 2473 ue_low = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW); 2474 ue_high = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HIGH); 2475 ue_low_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW_MASK); 2476 ue_high_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HI_MASK); 2477 2478 ue_low = (ue_low & ~ue_low_mask); 2479 ue_high = (ue_high & ~ue_high_mask); 2480 } 2481 2482 /* On certain platforms BE hardware can indicate spurious UEs. 2483 * Allow the h/w to stop working completely in case of a real UE. 2484 * Hence not setting the hw_error for UE detection. 2485 */ 2486 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2487 sc->hw_error = TRUE; 2488 device_printf(sc->dev, "Error detected in the card\n"); 2489 } 2490 2491 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2492 device_printf(sc->dev, 2493 "ERR: sliport status 0x%x\n", sliport_status); 2494 device_printf(sc->dev, 2495 "ERR: sliport error1 0x%x\n", sliport_err1); 2496 device_printf(sc->dev, 2497 "ERR: sliport error2 0x%x\n", sliport_err2); 2498 } 2499 2500 if (ue_low) { 2501 for (i = 0; ue_low; ue_low >>= 1, i++) { 2502 if (ue_low & 1) 2503 device_printf(sc->dev, "UE: %s bit set\n", 2504 ue_status_low_desc[i]); 2505 } 2506 } 2507 2508 if (ue_high) { 2509 for (i = 0; ue_high; ue_high >>= 1, i++) { 2510 if (ue_high & 1) 2511 device_printf(sc->dev, "UE: %s bit set\n", 2512 ue_status_hi_desc[i]); 2513 } 2514 } 2515 2516 } 2517 2518 2519 static void 2520 oce_local_timer(void *arg) 2521 { 2522 POCE_SOFTC sc = arg; 2523 int i = 0; 2524 2525 oce_detect_hw_error(sc); 2526 oce_refresh_nic_stats(sc); 2527 oce_refresh_queue_stats(sc); 2528 oce_mac_addr_set(sc); 2529 2530 /* TX Watch Dog*/ 2531 for (i = 0; i < sc->nwqs; i++) 2532 oce_tx_restart(sc, sc->wq[i]); 2533 2534 /* calculate and set the eq delay for optimal interrupt rate */ 2535 if (IS_BE(sc) || IS_SH(sc)) 2536 oce_eqd_set_periodic(sc); 2537 2538 callout_reset(&sc->timer, hz, oce_local_timer, sc); 2539 } 2540 2541 static void 2542 oce_tx_compl_clean(POCE_SOFTC sc) 2543 { 2544 struct oce_wq *wq; 2545 int i = 0, timeo = 0, num_wqes = 0; 2546 int pending_txqs = sc->nwqs; 2547 2548 /* Stop polling for compls when HW has been silent for 10ms or 2549 * hw_error or no outstanding completions expected 2550 */ 2551 do { 2552 pending_txqs = sc->nwqs; 2553 2554 for_all_wq_queues(sc, wq, i) { 2555 num_wqes = oce_wq_handler(wq); 2556 2557 if(num_wqes) 2558 timeo = 0; 2559 2560 if(!wq->ring->num_used) 2561 pending_txqs--; 2562 } 2563 2564 if (pending_txqs == 0 || ++timeo > 10 || sc->hw_error) 2565 break; 2566 2567 DELAY(1000); 2568 } while (TRUE); 2569 2570 for_all_wq_queues(sc, wq, i) { 2571 while(wq->ring->num_used) { 2572 LOCK(&wq->tx_compl_lock); 2573 oce_process_tx_completion(wq); 2574 UNLOCK(&wq->tx_compl_lock); 2575 } 2576 } 2577 2578 } 2579 2580 /* NOTE : This should only be called holding 2581 * DEVICE_LOCK. 2582 */ 2583 static void 2584 oce_if_deactivate(POCE_SOFTC sc) 2585 { 2586 int i; 2587 struct oce_rq *rq; 2588 struct oce_wq *wq; 2589 struct oce_eq *eq; 2590 2591 sc->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 2592 2593 oce_tx_compl_clean(sc); 2594 2595 /* Stop intrs and finish any bottom halves pending */ 2596 oce_hw_intr_disable(sc); 2597 2598 /* Since taskqueue_drain takes a Gaint Lock, We should not acquire 2599 any other lock. So unlock device lock and require after 2600 completing taskqueue_drain. 2601 */ 2602 UNLOCK(&sc->dev_lock); 2603 for (i = 0; i < sc->intr_count; i++) { 2604 if (sc->intrs[i].tq != NULL) { 2605 taskqueue_drain(sc->intrs[i].tq, &sc->intrs[i].task); 2606 } 2607 } 2608 LOCK(&sc->dev_lock); 2609 2610 /* Delete RX queue in card with flush param */ 2611 oce_stop_rx(sc); 2612 2613 /* Invalidate any pending cq and eq entries*/ 2614 for_all_evnt_queues(sc, eq, i) 2615 oce_drain_eq(eq); 2616 for_all_rq_queues(sc, rq, i) 2617 oce_drain_rq_cq(rq); 2618 for_all_wq_queues(sc, wq, i) 2619 oce_drain_wq_cq(wq); 2620 2621 /* But still we need to get MCC aync events. 2622 So enable intrs and also arm first EQ 2623 */ 2624 oce_hw_intr_enable(sc); 2625 oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE); 2626 2627 DELAY(10); 2628 } 2629 2630 2631 static void 2632 oce_if_activate(POCE_SOFTC sc) 2633 { 2634 struct oce_eq *eq; 2635 struct oce_rq *rq; 2636 struct oce_wq *wq; 2637 int i, rc = 0; 2638 2639 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; 2640 2641 oce_hw_intr_disable(sc); 2642 2643 oce_start_rx(sc); 2644 2645 for_all_rq_queues(sc, rq, i) { 2646 rc = oce_start_rq(rq); 2647 if (rc) 2648 device_printf(sc->dev, "Unable to start RX\n"); 2649 } 2650 2651 for_all_wq_queues(sc, wq, i) { 2652 rc = oce_start_wq(wq); 2653 if (rc) 2654 device_printf(sc->dev, "Unable to start TX\n"); 2655 } 2656 2657 2658 for_all_evnt_queues(sc, eq, i) 2659 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); 2660 2661 oce_hw_intr_enable(sc); 2662 2663 } 2664 2665 static void 2666 process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe) 2667 { 2668 /* Update Link status */ 2669 if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) == 2670 ASYNC_EVENT_LINK_UP) { 2671 sc->link_status = ASYNC_EVENT_LINK_UP; 2672 if_link_state_change(sc->ifp, LINK_STATE_UP); 2673 } else { 2674 sc->link_status = ASYNC_EVENT_LINK_DOWN; 2675 if_link_state_change(sc->ifp, LINK_STATE_DOWN); 2676 } 2677 } 2678 2679 2680 static void oce_async_grp5_osbmc_process(POCE_SOFTC sc, 2681 struct oce_async_evt_grp5_os2bmc *evt) 2682 { 2683 DW_SWAP(evt, sizeof(struct oce_async_evt_grp5_os2bmc)); 2684 if (evt->u.s.mgmt_enable) 2685 sc->flags |= OCE_FLAGS_OS2BMC; 2686 else 2687 return; 2688 2689 sc->bmc_filt_mask = evt->u.s.arp_filter; 2690 sc->bmc_filt_mask |= (evt->u.s.dhcp_client_filt << 1); 2691 sc->bmc_filt_mask |= (evt->u.s.dhcp_server_filt << 2); 2692 sc->bmc_filt_mask |= (evt->u.s.net_bios_filt << 3); 2693 sc->bmc_filt_mask |= (evt->u.s.bcast_filt << 4); 2694 sc->bmc_filt_mask |= (evt->u.s.ipv6_nbr_filt << 5); 2695 sc->bmc_filt_mask |= (evt->u.s.ipv6_ra_filt << 6); 2696 sc->bmc_filt_mask |= (evt->u.s.ipv6_ras_filt << 7); 2697 sc->bmc_filt_mask |= (evt->u.s.mcast_filt << 8); 2698 } 2699 2700 2701 static void oce_process_grp5_events(POCE_SOFTC sc, struct oce_mq_cqe *cqe) 2702 { 2703 struct oce_async_event_grp5_pvid_state *gcqe; 2704 struct oce_async_evt_grp5_os2bmc *bmccqe; 2705 2706 switch (cqe->u0.s.async_type) { 2707 case ASYNC_EVENT_PVID_STATE: 2708 /* GRP5 PVID */ 2709 gcqe = (struct oce_async_event_grp5_pvid_state *)cqe; 2710 if (gcqe->enabled) 2711 sc->pvid = gcqe->tag & VLAN_VID_MASK; 2712 else 2713 sc->pvid = 0; 2714 break; 2715 case ASYNC_EVENT_OS2BMC: 2716 bmccqe = (struct oce_async_evt_grp5_os2bmc *)cqe; 2717 oce_async_grp5_osbmc_process(sc, bmccqe); 2718 break; 2719 default: 2720 break; 2721 } 2722 } 2723 2724 /* Handle the Completion Queue for the Mailbox/Async notifications */ 2725 uint16_t 2726 oce_mq_handler(void *arg) 2727 { 2728 struct oce_mq *mq = (struct oce_mq *)arg; 2729 POCE_SOFTC sc = mq->parent; 2730 struct oce_cq *cq = mq->cq; 2731 int num_cqes = 0, evt_type = 0, optype = 0; 2732 struct oce_mq_cqe *cqe; 2733 struct oce_async_cqe_link_state *acqe; 2734 struct oce_async_event_qnq *dbgcqe; 2735 2736 2737 bus_dmamap_sync(cq->ring->dma.tag, 2738 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2739 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe); 2740 2741 while (cqe->u0.dw[3]) { 2742 DW_SWAP((uint32_t *) cqe, sizeof(oce_mq_cqe)); 2743 if (cqe->u0.s.async_event) { 2744 evt_type = cqe->u0.s.event_type; 2745 optype = cqe->u0.s.async_type; 2746 if (evt_type == ASYNC_EVENT_CODE_LINK_STATE) { 2747 /* Link status evt */ 2748 acqe = (struct oce_async_cqe_link_state *)cqe; 2749 process_link_state(sc, acqe); 2750 } else if (evt_type == ASYNC_EVENT_GRP5) { 2751 oce_process_grp5_events(sc, cqe); 2752 } else if (evt_type == ASYNC_EVENT_CODE_DEBUG && 2753 optype == ASYNC_EVENT_DEBUG_QNQ) { 2754 dbgcqe = (struct oce_async_event_qnq *)cqe; 2755 if(dbgcqe->valid) 2756 sc->qnqid = dbgcqe->vlan_tag; 2757 sc->qnq_debug_event = TRUE; 2758 } 2759 } 2760 cqe->u0.dw[3] = 0; 2761 RING_GET(cq->ring, 1); 2762 bus_dmamap_sync(cq->ring->dma.tag, 2763 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2764 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe); 2765 num_cqes++; 2766 } 2767 2768 if (num_cqes) 2769 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); 2770 2771 return 0; 2772 } 2773 2774 2775 static void 2776 setup_max_queues_want(POCE_SOFTC sc) 2777 { 2778 /* Check if it is FLEX machine. Is so dont use RSS */ 2779 if ((sc->function_mode & FNM_FLEX10_MODE) || 2780 (sc->function_mode & FNM_UMC_MODE) || 2781 (sc->function_mode & FNM_VNIC_MODE) || 2782 (!is_rss_enabled(sc)) || 2783 IS_BE2(sc)) { 2784 sc->nrqs = 1; 2785 sc->nwqs = 1; 2786 } else { 2787 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1; 2788 sc->nwqs = MIN(OCE_NCPUS, sc->nrssqs); 2789 } 2790 2791 if (IS_BE2(sc) && is_rss_enabled(sc)) 2792 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1; 2793 } 2794 2795 2796 static void 2797 update_queues_got(POCE_SOFTC sc) 2798 { 2799 if (is_rss_enabled(sc)) { 2800 sc->nrqs = sc->intr_count + 1; 2801 sc->nwqs = sc->intr_count; 2802 } else { 2803 sc->nrqs = 1; 2804 sc->nwqs = 1; 2805 } 2806 2807 if (IS_BE2(sc)) 2808 sc->nwqs = 1; 2809 } 2810 2811 static int 2812 oce_check_ipv6_ext_hdr(struct mbuf *m) 2813 { 2814 struct ether_header *eh = mtod(m, struct ether_header *); 2815 caddr_t m_datatemp = m->m_data; 2816 2817 if (eh->ether_type == htons(ETHERTYPE_IPV6)) { 2818 m->m_data += sizeof(struct ether_header); 2819 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); 2820 2821 if((ip6->ip6_nxt != IPPROTO_TCP) && \ 2822 (ip6->ip6_nxt != IPPROTO_UDP)){ 2823 struct ip6_ext *ip6e = NULL; 2824 m->m_data += sizeof(struct ip6_hdr); 2825 2826 ip6e = (struct ip6_ext *) mtod(m, struct ip6_ext *); 2827 if(ip6e->ip6e_len == 0xff) { 2828 m->m_data = m_datatemp; 2829 return TRUE; 2830 } 2831 } 2832 m->m_data = m_datatemp; 2833 } 2834 return FALSE; 2835 } 2836 2837 static int 2838 is_be3_a1(POCE_SOFTC sc) 2839 { 2840 if((sc->flags & OCE_FLAGS_BE3) && ((sc->asic_revision & 0xFF) < 2)) { 2841 return TRUE; 2842 } 2843 return FALSE; 2844 } 2845 2846 static struct mbuf * 2847 oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete) 2848 { 2849 uint16_t vlan_tag = 0; 2850 2851 if(!M_WRITABLE(m)) 2852 return NULL; 2853 2854 /* Embed vlan tag in the packet if it is not part of it */ 2855 if(m->m_flags & M_VLANTAG) { 2856 vlan_tag = EVL_VLANOFTAG(m->m_pkthdr.ether_vtag); 2857 m->m_flags &= ~M_VLANTAG; 2858 } 2859 2860 /* if UMC, ignore vlan tag insertion and instead insert pvid */ 2861 if(sc->pvid) { 2862 if(!vlan_tag) 2863 vlan_tag = sc->pvid; 2864 if (complete) 2865 *complete = FALSE; 2866 } 2867 2868 if(vlan_tag) { 2869 m = ether_vlanencap(m, vlan_tag); 2870 } 2871 2872 if(sc->qnqid) { 2873 m = ether_vlanencap(m, sc->qnqid); 2874 2875 if (complete) 2876 *complete = FALSE; 2877 } 2878 return m; 2879 } 2880 2881 static int 2882 oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m) 2883 { 2884 if(is_be3_a1(sc) && IS_QNQ_OR_UMC(sc) && \ 2885 oce_check_ipv6_ext_hdr(m)) { 2886 return TRUE; 2887 } 2888 return FALSE; 2889 } 2890 2891 static void 2892 oce_get_config(POCE_SOFTC sc) 2893 { 2894 int rc = 0; 2895 uint32_t max_rss = 0; 2896 2897 if ((IS_BE(sc) || IS_SH(sc)) && (!sc->be3_native)) 2898 max_rss = OCE_LEGACY_MODE_RSS; 2899 else 2900 max_rss = OCE_MAX_RSS; 2901 2902 if (!IS_BE(sc)) { 2903 rc = oce_get_profile_config(sc, max_rss); 2904 if (rc) { 2905 sc->nwqs = OCE_MAX_WQ; 2906 sc->nrssqs = max_rss; 2907 sc->nrqs = sc->nrssqs + 1; 2908 } 2909 } 2910 else { /* For BE3 don't rely on fw for determining the resources */ 2911 sc->nrssqs = max_rss; 2912 sc->nrqs = sc->nrssqs + 1; 2913 sc->nwqs = OCE_MAX_WQ; 2914 sc->max_vlans = MAX_VLANFILTER_SIZE; 2915 } 2916 } 2917 2918 static void 2919 oce_rdma_close(void) 2920 { 2921 if (oce_rdma_if != NULL) { 2922 oce_rdma_if = NULL; 2923 } 2924 } 2925 2926 static void 2927 oce_get_mac_addr(POCE_SOFTC sc, uint8_t *macaddr) 2928 { 2929 memcpy(macaddr, sc->macaddr.mac_addr, 6); 2930 } 2931 2932 int 2933 oce_register_rdma(POCE_RDMA_INFO rdma_info, POCE_RDMA_IF rdma_if) 2934 { 2935 POCE_SOFTC sc; 2936 struct oce_dev_info di; 2937 int i; 2938 2939 if ((rdma_info == NULL) || (rdma_if == NULL)) { 2940 return -EINVAL; 2941 } 2942 2943 if ((rdma_info->size != OCE_RDMA_INFO_SIZE) || 2944 (rdma_if->size != OCE_RDMA_IF_SIZE)) { 2945 return -ENXIO; 2946 } 2947 2948 rdma_info->close = oce_rdma_close; 2949 rdma_info->mbox_post = oce_mbox_post; 2950 rdma_info->common_req_hdr_init = mbx_common_req_hdr_init; 2951 rdma_info->get_mac_addr = oce_get_mac_addr; 2952 2953 oce_rdma_if = rdma_if; 2954 2955 sc = softc_head; 2956 while (sc != NULL) { 2957 if (oce_rdma_if->announce != NULL) { 2958 memset(&di, 0, sizeof(di)); 2959 di.dev = sc->dev; 2960 di.softc = sc; 2961 di.ifp = sc->ifp; 2962 di.db_bhandle = sc->db_bhandle; 2963 di.db_btag = sc->db_btag; 2964 di.db_page_size = 4096; 2965 if (sc->flags & OCE_FLAGS_USING_MSIX) { 2966 di.intr_mode = OCE_INTERRUPT_MODE_MSIX; 2967 } else if (sc->flags & OCE_FLAGS_USING_MSI) { 2968 di.intr_mode = OCE_INTERRUPT_MODE_MSI; 2969 } else { 2970 di.intr_mode = OCE_INTERRUPT_MODE_INTX; 2971 } 2972 di.dev_family = OCE_GEN2_FAMILY; // fixme: must detect skyhawk 2973 if (di.intr_mode != OCE_INTERRUPT_MODE_INTX) { 2974 di.msix.num_vectors = sc->intr_count + sc->roce_intr_count; 2975 di.msix.start_vector = sc->intr_count; 2976 for (i=0; i<di.msix.num_vectors; i++) { 2977 di.msix.vector_list[i] = sc->intrs[i].vector; 2978 } 2979 } else { 2980 } 2981 memcpy(di.mac_addr, sc->macaddr.mac_addr, 6); 2982 di.vendor_id = pci_get_vendor(sc->dev); 2983 di.dev_id = pci_get_device(sc->dev); 2984 2985 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 2986 di.flags |= OCE_RDMA_INFO_RDMA_SUPPORTED; 2987 } 2988 2989 rdma_if->announce(&di); 2990 sc = sc->next; 2991 } 2992 } 2993 2994 return 0; 2995 } 2996 2997 static void 2998 oce_read_env_variables( POCE_SOFTC sc ) 2999 { 3000 char *value = NULL; 3001 int rc = 0; 3002 3003 /* read if user wants to enable hwlro or swlro */ 3004 //value = getenv("oce_enable_hwlro"); 3005 if(value && IS_SH(sc)) { 3006 sc->enable_hwlro = strtol(value, NULL, 10); 3007 if(sc->enable_hwlro) { 3008 rc = oce_mbox_nic_query_lro_capabilities(sc, NULL, NULL); 3009 if(rc) { 3010 device_printf(sc->dev, "no hardware lro support\n"); 3011 device_printf(sc->dev, "software lro enabled\n"); 3012 sc->enable_hwlro = 0; 3013 }else { 3014 device_printf(sc->dev, "hardware lro enabled\n"); 3015 oce_max_rsp_handled = 32; 3016 } 3017 }else { 3018 device_printf(sc->dev, "software lro enabled\n"); 3019 } 3020 }else { 3021 sc->enable_hwlro = 0; 3022 } 3023 3024 /* read mbuf size */ 3025 //value = getenv("oce_rq_buf_size"); 3026 if(value && IS_SH(sc)) { 3027 oce_rq_buf_size = strtol(value, NULL, 10); 3028 switch(oce_rq_buf_size) { 3029 case 2048: 3030 case 4096: 3031 case 9216: 3032 case 16384: 3033 break; 3034 3035 default: 3036 device_printf(sc->dev, " Supported oce_rq_buf_size values are 2K, 4K, 9K, 16K \n"); 3037 oce_rq_buf_size = 2048; 3038 } 3039 } 3040 3041 return; 3042 } 3043