1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (C) 2013 Emulex 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Emulex Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 * 33 * Contact Information: 34 * freebsd-drivers@emulex.com 35 * 36 * Emulex 37 * 3333 Susan Street 38 * Costa Mesa, CA 92626 39 */ 40 41 /* $FreeBSD$ */ 42 43 #include "opt_inet6.h" 44 #include "opt_inet.h" 45 46 #include "oce_if.h" 47 #include "oce_user.h" 48 49 #define is_tso_pkt(m) (m->m_pkthdr.csum_flags & CSUM_TSO) 50 51 /* UE Status Low CSR */ 52 static char *ue_status_low_desc[] = { 53 "CEV", 54 "CTX", 55 "DBUF", 56 "ERX", 57 "Host", 58 "MPU", 59 "NDMA", 60 "PTC ", 61 "RDMA ", 62 "RXF ", 63 "RXIPS ", 64 "RXULP0 ", 65 "RXULP1 ", 66 "RXULP2 ", 67 "TIM ", 68 "TPOST ", 69 "TPRE ", 70 "TXIPS ", 71 "TXULP0 ", 72 "TXULP1 ", 73 "UC ", 74 "WDMA ", 75 "TXULP2 ", 76 "HOST1 ", 77 "P0_OB_LINK ", 78 "P1_OB_LINK ", 79 "HOST_GPIO ", 80 "MBOX ", 81 "AXGMAC0", 82 "AXGMAC1", 83 "JTAG", 84 "MPU_INTPEND" 85 }; 86 87 /* UE Status High CSR */ 88 static char *ue_status_hi_desc[] = { 89 "LPCMEMHOST", 90 "MGMT_MAC", 91 "PCS0ONLINE", 92 "MPU_IRAM", 93 "PCS1ONLINE", 94 "PCTL0", 95 "PCTL1", 96 "PMEM", 97 "RR", 98 "TXPB", 99 "RXPP", 100 "XAUI", 101 "TXP", 102 "ARM", 103 "IPC", 104 "HOST2", 105 "HOST3", 106 "HOST4", 107 "HOST5", 108 "HOST6", 109 "HOST7", 110 "HOST8", 111 "HOST9", 112 "NETC", 113 "Unknown", 114 "Unknown", 115 "Unknown", 116 "Unknown", 117 "Unknown", 118 "Unknown", 119 "Unknown", 120 "Unknown" 121 }; 122 123 struct oce_common_cqe_info{ 124 uint8_t vtp:1; 125 uint8_t l4_cksum_pass:1; 126 uint8_t ip_cksum_pass:1; 127 uint8_t ipv6_frame:1; 128 uint8_t qnq:1; 129 uint8_t rsvd:3; 130 uint8_t num_frags; 131 uint16_t pkt_size; 132 uint16_t vtag; 133 }; 134 135 /* Driver entry points prototypes */ 136 static int oce_probe(device_t dev); 137 static int oce_attach(device_t dev); 138 static int oce_detach(device_t dev); 139 static int oce_shutdown(device_t dev); 140 static int oce_ioctl(if_t ifp, u_long command, caddr_t data); 141 static void oce_init(void *xsc); 142 static int oce_multiq_start(if_t ifp, struct mbuf *m); 143 static void oce_multiq_flush(if_t ifp); 144 145 /* Driver interrupt routines protypes */ 146 static void oce_intr(void *arg, int pending); 147 static int oce_setup_intr(POCE_SOFTC sc); 148 static int oce_fast_isr(void *arg); 149 static int oce_alloc_intr(POCE_SOFTC sc, int vector, 150 void (*isr) (void *arg, int pending)); 151 152 /* Media callbacks prototypes */ 153 static void oce_media_status(if_t ifp, struct ifmediareq *req); 154 static int oce_media_change(if_t ifp); 155 156 /* Transmit routines prototypes */ 157 static int oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index); 158 static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq); 159 static void oce_process_tx_completion(struct oce_wq *wq); 160 static int oce_multiq_transmit(if_t ifp, struct mbuf *m, 161 struct oce_wq *wq); 162 163 /* Receive routines prototypes */ 164 static int oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe); 165 static int oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe); 166 static void oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe); 167 static void oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq); 168 static uint16_t oce_rq_handler_lro(void *arg); 169 static void oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2); 170 static void oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2); 171 static void oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m); 172 173 /* Helper function prototypes in this file */ 174 static int oce_attach_ifp(POCE_SOFTC sc); 175 static void oce_add_vlan(void *arg, if_t ifp, uint16_t vtag); 176 static void oce_del_vlan(void *arg, if_t ifp, uint16_t vtag); 177 static int oce_vid_config(POCE_SOFTC sc); 178 static void oce_mac_addr_set(POCE_SOFTC sc); 179 static int oce_handle_passthrough(if_t ifp, caddr_t data); 180 static void oce_local_timer(void *arg); 181 static void oce_if_deactivate(POCE_SOFTC sc); 182 static void oce_if_activate(POCE_SOFTC sc); 183 static void setup_max_queues_want(POCE_SOFTC sc); 184 static void update_queues_got(POCE_SOFTC sc); 185 static void process_link_state(POCE_SOFTC sc, 186 struct oce_async_cqe_link_state *acqe); 187 static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m); 188 static void oce_get_config(POCE_SOFTC sc); 189 static struct mbuf *oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete); 190 static void oce_read_env_variables(POCE_SOFTC sc); 191 192 /* IP specific */ 193 #if defined(INET6) || defined(INET) 194 static int oce_init_lro(POCE_SOFTC sc); 195 static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp); 196 #endif 197 198 static device_method_t oce_dispatch[] = { 199 DEVMETHOD(device_probe, oce_probe), 200 DEVMETHOD(device_attach, oce_attach), 201 DEVMETHOD(device_detach, oce_detach), 202 DEVMETHOD(device_shutdown, oce_shutdown), 203 204 DEVMETHOD_END 205 }; 206 207 static driver_t oce_driver = { 208 "oce", 209 oce_dispatch, 210 sizeof(OCE_SOFTC) 211 }; 212 213 /* global vars */ 214 const char component_revision[32] = {"///" COMPONENT_REVISION "///"}; 215 216 /* Module capabilites and parameters */ 217 uint32_t oce_max_rsp_handled = OCE_MAX_RSP_HANDLED; 218 uint32_t oce_enable_rss = OCE_MODCAP_RSS; 219 uint32_t oce_rq_buf_size = 2048; 220 221 TUNABLE_INT("hw.oce.max_rsp_handled", &oce_max_rsp_handled); 222 TUNABLE_INT("hw.oce.enable_rss", &oce_enable_rss); 223 224 /* Supported devices table */ 225 static uint32_t supportedDevices[] = { 226 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE2, 227 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE3, 228 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_BE3, 229 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201, 230 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201_VF, 231 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_SH 232 }; 233 234 DRIVER_MODULE(oce, pci, oce_driver, 0, 0); 235 MODULE_PNP_INFO("W32:vendor/device", pci, oce, supportedDevices, 236 nitems(supportedDevices)); 237 MODULE_DEPEND(oce, pci, 1, 1, 1); 238 MODULE_DEPEND(oce, ether, 1, 1, 1); 239 MODULE_VERSION(oce, 1); 240 241 POCE_SOFTC softc_head = NULL; 242 POCE_SOFTC softc_tail = NULL; 243 244 struct oce_rdma_if *oce_rdma_if = NULL; 245 246 /***************************************************************************** 247 * Driver entry points functions * 248 *****************************************************************************/ 249 250 static int 251 oce_probe(device_t dev) 252 { 253 uint16_t vendor = 0; 254 uint16_t device = 0; 255 int i = 0; 256 char str[256] = {0}; 257 POCE_SOFTC sc; 258 259 sc = device_get_softc(dev); 260 bzero(sc, sizeof(OCE_SOFTC)); 261 sc->dev = dev; 262 263 vendor = pci_get_vendor(dev); 264 device = pci_get_device(dev); 265 266 for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint32_t)); i++) { 267 if (vendor == ((supportedDevices[i] >> 16) & 0xffff)) { 268 if (device == (supportedDevices[i] & 0xffff)) { 269 sprintf(str, "%s:%s", "Emulex CNA NIC function", 270 component_revision); 271 device_set_desc_copy(dev, str); 272 273 switch (device) { 274 case PCI_PRODUCT_BE2: 275 sc->flags |= OCE_FLAGS_BE2; 276 break; 277 case PCI_PRODUCT_BE3: 278 sc->flags |= OCE_FLAGS_BE3; 279 break; 280 case PCI_PRODUCT_XE201: 281 case PCI_PRODUCT_XE201_VF: 282 sc->flags |= OCE_FLAGS_XE201; 283 break; 284 case PCI_PRODUCT_SH: 285 sc->flags |= OCE_FLAGS_SH; 286 break; 287 default: 288 return ENXIO; 289 } 290 return BUS_PROBE_DEFAULT; 291 } 292 } 293 } 294 295 return ENXIO; 296 } 297 298 static int 299 oce_attach(device_t dev) 300 { 301 POCE_SOFTC sc; 302 int rc = 0; 303 304 sc = device_get_softc(dev); 305 306 rc = oce_hw_pci_alloc(sc); 307 if (rc) 308 return rc; 309 310 sc->tx_ring_size = OCE_TX_RING_SIZE; 311 sc->rx_ring_size = OCE_RX_RING_SIZE; 312 /* receive fragment size should be multiple of 2K */ 313 sc->rq_frag_size = ((oce_rq_buf_size / 2048) * 2048); 314 sc->flow_control = OCE_DEFAULT_FLOW_CONTROL; 315 sc->promisc = OCE_DEFAULT_PROMISCUOUS; 316 317 LOCK_CREATE(&sc->bmbx_lock, "Mailbox_lock"); 318 LOCK_CREATE(&sc->dev_lock, "Device_lock"); 319 320 /* initialise the hardware */ 321 rc = oce_hw_init(sc); 322 if (rc) 323 goto pci_res_free; 324 325 oce_read_env_variables(sc); 326 327 oce_get_config(sc); 328 329 setup_max_queues_want(sc); 330 331 rc = oce_setup_intr(sc); 332 if (rc) 333 goto mbox_free; 334 335 rc = oce_queue_init_all(sc); 336 if (rc) 337 goto intr_free; 338 339 rc = oce_attach_ifp(sc); 340 if (rc) 341 goto queues_free; 342 343 #if defined(INET6) || defined(INET) 344 rc = oce_init_lro(sc); 345 if (rc) 346 goto ifp_free; 347 #endif 348 349 rc = oce_hw_start(sc); 350 if (rc) 351 goto lro_free; 352 353 sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 354 oce_add_vlan, sc, EVENTHANDLER_PRI_FIRST); 355 sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 356 oce_del_vlan, sc, EVENTHANDLER_PRI_FIRST); 357 358 rc = oce_stats_init(sc); 359 if (rc) 360 goto vlan_free; 361 362 oce_add_sysctls(sc); 363 364 callout_init(&sc->timer, CALLOUT_MPSAFE); 365 rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc); 366 if (rc) 367 goto stats_free; 368 369 sc->next =NULL; 370 if (softc_tail != NULL) { 371 softc_tail->next = sc; 372 } else { 373 softc_head = sc; 374 } 375 softc_tail = sc; 376 377 gone_in_dev(dev, 15, "relatively uncommon 10GbE NIC"); 378 379 return 0; 380 381 stats_free: 382 callout_drain(&sc->timer); 383 oce_stats_free(sc); 384 vlan_free: 385 if (sc->vlan_attach) 386 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); 387 if (sc->vlan_detach) 388 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); 389 oce_hw_intr_disable(sc); 390 lro_free: 391 #if defined(INET6) || defined(INET) 392 oce_free_lro(sc); 393 ifp_free: 394 #endif 395 ether_ifdetach(sc->ifp); 396 if_free(sc->ifp); 397 queues_free: 398 oce_queue_release_all(sc); 399 intr_free: 400 oce_intr_free(sc); 401 mbox_free: 402 oce_dma_free(sc, &sc->bsmbx); 403 pci_res_free: 404 oce_hw_pci_free(sc); 405 LOCK_DESTROY(&sc->dev_lock); 406 LOCK_DESTROY(&sc->bmbx_lock); 407 return rc; 408 409 } 410 411 static int 412 oce_detach(device_t dev) 413 { 414 POCE_SOFTC sc = device_get_softc(dev); 415 POCE_SOFTC poce_sc_tmp, *ppoce_sc_tmp1, poce_sc_tmp2 = NULL; 416 417 poce_sc_tmp = softc_head; 418 ppoce_sc_tmp1 = &softc_head; 419 while (poce_sc_tmp != NULL) { 420 if (poce_sc_tmp == sc) { 421 *ppoce_sc_tmp1 = sc->next; 422 if (sc->next == NULL) { 423 softc_tail = poce_sc_tmp2; 424 } 425 break; 426 } 427 poce_sc_tmp2 = poce_sc_tmp; 428 ppoce_sc_tmp1 = &poce_sc_tmp->next; 429 poce_sc_tmp = poce_sc_tmp->next; 430 } 431 432 LOCK(&sc->dev_lock); 433 oce_if_deactivate(sc); 434 UNLOCK(&sc->dev_lock); 435 436 callout_drain(&sc->timer); 437 438 if (sc->vlan_attach != NULL) 439 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); 440 if (sc->vlan_detach != NULL) 441 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); 442 443 ether_ifdetach(sc->ifp); 444 445 if_free(sc->ifp); 446 447 oce_hw_shutdown(sc); 448 449 bus_generic_detach(dev); 450 451 return 0; 452 } 453 454 static int 455 oce_shutdown(device_t dev) 456 { 457 int rc; 458 459 rc = oce_detach(dev); 460 461 return rc; 462 } 463 464 static int 465 oce_ioctl(if_t ifp, u_long command, caddr_t data) 466 { 467 struct ifreq *ifr = (struct ifreq *)data; 468 POCE_SOFTC sc = if_getsoftc(ifp); 469 struct ifi2creq i2c; 470 uint8_t offset = 0; 471 int rc = 0; 472 uint32_t u; 473 474 switch (command) { 475 case SIOCGIFMEDIA: 476 rc = ifmedia_ioctl(ifp, ifr, &sc->media, command); 477 break; 478 479 case SIOCSIFMTU: 480 if (ifr->ifr_mtu > OCE_MAX_MTU) 481 rc = EINVAL; 482 else 483 if_setmtu(ifp, ifr->ifr_mtu); 484 break; 485 486 case SIOCSIFFLAGS: 487 if (if_getflags(ifp) & IFF_UP) { 488 if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) { 489 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0); 490 oce_init(sc); 491 } 492 device_printf(sc->dev, "Interface Up\n"); 493 } else { 494 LOCK(&sc->dev_lock); 495 496 if_setdrvflagbits(sc->ifp, 0, 497 IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 498 oce_if_deactivate(sc); 499 500 UNLOCK(&sc->dev_lock); 501 502 device_printf(sc->dev, "Interface Down\n"); 503 } 504 505 if ((if_getflags(ifp) & IFF_PROMISC) && !sc->promisc) { 506 if (!oce_rxf_set_promiscuous(sc, (1 | (1 << 1)))) 507 sc->promisc = TRUE; 508 } else if (!(if_getflags(ifp) & IFF_PROMISC) && sc->promisc) { 509 if (!oce_rxf_set_promiscuous(sc, 0)) 510 sc->promisc = FALSE; 511 } 512 513 break; 514 515 case SIOCADDMULTI: 516 case SIOCDELMULTI: 517 rc = oce_hw_update_multicast(sc); 518 if (rc) 519 device_printf(sc->dev, 520 "Update multicast address failed\n"); 521 break; 522 523 case SIOCSIFCAP: 524 u = ifr->ifr_reqcap ^ if_getcapenable(ifp); 525 526 if (u & IFCAP_TXCSUM) { 527 if_togglecapenable(ifp, IFCAP_TXCSUM); 528 if_togglehwassist(ifp, (CSUM_TCP | CSUM_UDP | CSUM_IP)); 529 530 if (IFCAP_TSO & if_getcapenable(ifp) && 531 !(IFCAP_TXCSUM & if_getcapenable(ifp))) { 532 u &= ~IFCAP_TSO; 533 if_setcapenablebit(ifp, 0, IFCAP_TSO); 534 if_sethwassistbits(ifp, 0, CSUM_TSO); 535 if_printf(ifp, 536 "TSO disabled due to -txcsum.\n"); 537 } 538 } 539 540 if (u & IFCAP_RXCSUM) 541 if_togglecapenable(ifp, IFCAP_RXCSUM); 542 543 if (u & IFCAP_TSO4) { 544 if_togglecapenable(ifp, IFCAP_TSO4); 545 546 if (IFCAP_TSO & if_getcapenable(ifp)) { 547 if (IFCAP_TXCSUM & if_getcapenable(ifp)) 548 if_sethwassistbits(ifp, CSUM_TSO, 0); 549 else { 550 if_setcapenablebit(ifp, 0, IFCAP_TSO); 551 if_sethwassistbits(ifp, 0, CSUM_TSO); 552 if_printf(ifp, 553 "Enable txcsum first.\n"); 554 rc = EAGAIN; 555 } 556 } else 557 if_sethwassistbits(ifp, 0, CSUM_TSO); 558 } 559 560 if (u & IFCAP_VLAN_HWTAGGING) 561 if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING); 562 563 if (u & IFCAP_VLAN_HWFILTER) { 564 if_togglecapenable(ifp, IFCAP_VLAN_HWFILTER); 565 oce_vid_config(sc); 566 } 567 #if defined(INET6) || defined(INET) 568 if (u & IFCAP_LRO) { 569 if_togglecapenable(ifp, IFCAP_LRO); 570 if(sc->enable_hwlro) { 571 if(if_getcapenable(ifp) & IFCAP_LRO) { 572 rc = oce_mbox_nic_set_iface_lro_config(sc, 1); 573 }else { 574 rc = oce_mbox_nic_set_iface_lro_config(sc, 0); 575 } 576 } 577 } 578 #endif 579 580 break; 581 582 case SIOCGI2C: 583 rc = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c)); 584 if (rc) 585 break; 586 587 if (i2c.dev_addr == PAGE_NUM_A0) { 588 offset = i2c.offset; 589 } else if (i2c.dev_addr == PAGE_NUM_A2) { 590 offset = TRANSCEIVER_A0_SIZE + i2c.offset; 591 } else { 592 rc = EINVAL; 593 break; 594 } 595 596 if (i2c.len > sizeof(i2c.data) || 597 i2c.len + offset > sizeof(sfp_vpd_dump_buffer)) { 598 rc = EINVAL; 599 break; 600 } 601 602 rc = oce_mbox_read_transrecv_data(sc, i2c.dev_addr); 603 if (rc) { 604 rc = -rc; 605 break; 606 } 607 608 memcpy(&i2c.data[0], &sfp_vpd_dump_buffer[offset], i2c.len); 609 610 rc = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c)); 611 break; 612 613 case SIOCGPRIVATE_0: 614 rc = priv_check(curthread, PRIV_DRIVER); 615 if (rc != 0) 616 break; 617 rc = oce_handle_passthrough(ifp, data); 618 break; 619 default: 620 rc = ether_ioctl(ifp, command, data); 621 break; 622 } 623 624 return rc; 625 } 626 627 static void 628 oce_init(void *arg) 629 { 630 POCE_SOFTC sc = arg; 631 632 LOCK(&sc->dev_lock); 633 634 if (if_getflags(sc->ifp) & IFF_UP) { 635 oce_if_deactivate(sc); 636 oce_if_activate(sc); 637 } 638 639 UNLOCK(&sc->dev_lock); 640 641 } 642 643 static int 644 oce_multiq_start(if_t ifp, struct mbuf *m) 645 { 646 POCE_SOFTC sc = if_getsoftc(ifp); 647 struct oce_wq *wq = NULL; 648 int queue_index = 0; 649 int status = 0; 650 651 if (!sc->link_status) 652 return ENXIO; 653 654 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 655 queue_index = m->m_pkthdr.flowid % sc->nwqs; 656 657 wq = sc->wq[queue_index]; 658 659 LOCK(&wq->tx_lock); 660 status = oce_multiq_transmit(ifp, m, wq); 661 UNLOCK(&wq->tx_lock); 662 663 return status; 664 665 } 666 667 static void 668 oce_multiq_flush(if_t ifp) 669 { 670 POCE_SOFTC sc = if_getsoftc(ifp); 671 struct mbuf *m; 672 int i = 0; 673 674 for (i = 0; i < sc->nwqs; i++) { 675 while ((m = buf_ring_dequeue_sc(sc->wq[i]->br)) != NULL) 676 m_freem(m); 677 } 678 if_qflush(ifp); 679 } 680 681 /***************************************************************************** 682 * Driver interrupt routines functions * 683 *****************************************************************************/ 684 685 static void 686 oce_intr(void *arg, int pending) 687 { 688 689 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg; 690 POCE_SOFTC sc = ii->sc; 691 struct oce_eq *eq = ii->eq; 692 struct oce_eqe *eqe; 693 struct oce_cq *cq = NULL; 694 int i, num_eqes = 0; 695 696 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, 697 BUS_DMASYNC_POSTWRITE); 698 do { 699 eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe); 700 if (eqe->evnt == 0) 701 break; 702 eqe->evnt = 0; 703 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, 704 BUS_DMASYNC_POSTWRITE); 705 RING_GET(eq->ring, 1); 706 num_eqes++; 707 708 } while (TRUE); 709 710 if (!num_eqes) 711 goto eq_arm; /* Spurious */ 712 713 /* Clear EQ entries, but dont arm */ 714 oce_arm_eq(sc, eq->eq_id, num_eqes, FALSE, FALSE); 715 716 /* Process TX, RX and MCC. But dont arm CQ*/ 717 for (i = 0; i < eq->cq_valid; i++) { 718 cq = eq->cq[i]; 719 (*cq->cq_handler)(cq->cb_arg); 720 } 721 722 /* Arm all cqs connected to this EQ */ 723 for (i = 0; i < eq->cq_valid; i++) { 724 cq = eq->cq[i]; 725 oce_arm_cq(sc, cq->cq_id, 0, TRUE); 726 } 727 728 eq_arm: 729 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); 730 731 return; 732 } 733 734 static int 735 oce_setup_intr(POCE_SOFTC sc) 736 { 737 int rc = 0, use_intx = 0; 738 int vector = 0, req_vectors = 0; 739 int tot_req_vectors, tot_vectors; 740 741 if (is_rss_enabled(sc)) 742 req_vectors = MAX((sc->nrqs - 1), sc->nwqs); 743 else 744 req_vectors = 1; 745 746 tot_req_vectors = req_vectors; 747 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 748 if (req_vectors > 1) { 749 tot_req_vectors += OCE_RDMA_VECTORS; 750 sc->roce_intr_count = OCE_RDMA_VECTORS; 751 } 752 } 753 754 if (sc->flags & OCE_FLAGS_MSIX_CAPABLE) { 755 sc->intr_count = req_vectors; 756 tot_vectors = tot_req_vectors; 757 rc = pci_alloc_msix(sc->dev, &tot_vectors); 758 if (rc != 0) { 759 use_intx = 1; 760 pci_release_msi(sc->dev); 761 } else { 762 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 763 if (tot_vectors < tot_req_vectors) { 764 if (sc->intr_count < (2 * OCE_RDMA_VECTORS)) { 765 sc->roce_intr_count = (tot_vectors / 2); 766 } 767 sc->intr_count = tot_vectors - sc->roce_intr_count; 768 } 769 } else { 770 sc->intr_count = tot_vectors; 771 } 772 sc->flags |= OCE_FLAGS_USING_MSIX; 773 } 774 } else 775 use_intx = 1; 776 777 if (use_intx) 778 sc->intr_count = 1; 779 780 /* Scale number of queues based on intr we got */ 781 update_queues_got(sc); 782 783 if (use_intx) { 784 device_printf(sc->dev, "Using legacy interrupt\n"); 785 rc = oce_alloc_intr(sc, vector, oce_intr); 786 if (rc) 787 goto error; 788 } else { 789 for (; vector < sc->intr_count; vector++) { 790 rc = oce_alloc_intr(sc, vector, oce_intr); 791 if (rc) 792 goto error; 793 } 794 } 795 796 return 0; 797 error: 798 oce_intr_free(sc); 799 return rc; 800 } 801 802 static int 803 oce_fast_isr(void *arg) 804 { 805 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg; 806 POCE_SOFTC sc = ii->sc; 807 808 if (ii->eq == NULL) 809 return FILTER_STRAY; 810 811 oce_arm_eq(sc, ii->eq->eq_id, 0, FALSE, TRUE); 812 813 taskqueue_enqueue(ii->tq, &ii->task); 814 815 ii->eq->intr++; 816 817 return FILTER_HANDLED; 818 } 819 820 static int 821 oce_alloc_intr(POCE_SOFTC sc, int vector, void (*isr) (void *arg, int pending)) 822 { 823 POCE_INTR_INFO ii; 824 int rc = 0, rr; 825 826 if (vector >= OCE_MAX_EQ) 827 return (EINVAL); 828 829 ii = &sc->intrs[vector]; 830 831 /* Set the resource id for the interrupt. 832 * MSIx is vector + 1 for the resource id, 833 * INTx is 0 for the resource id. 834 */ 835 if (sc->flags & OCE_FLAGS_USING_MSIX) 836 rr = vector + 1; 837 else 838 rr = 0; 839 ii->intr_res = bus_alloc_resource_any(sc->dev, 840 SYS_RES_IRQ, 841 &rr, RF_ACTIVE|RF_SHAREABLE); 842 ii->irq_rr = rr; 843 if (ii->intr_res == NULL) { 844 device_printf(sc->dev, 845 "Could not allocate interrupt\n"); 846 rc = ENXIO; 847 return rc; 848 } 849 850 TASK_INIT(&ii->task, 0, isr, ii); 851 ii->vector = vector; 852 sprintf(ii->task_name, "oce_task[%d]", ii->vector); 853 ii->tq = taskqueue_create_fast(ii->task_name, 854 M_NOWAIT, 855 taskqueue_thread_enqueue, 856 &ii->tq); 857 taskqueue_start_threads(&ii->tq, 1, PI_NET, "%s taskq", 858 device_get_nameunit(sc->dev)); 859 860 ii->sc = sc; 861 rc = bus_setup_intr(sc->dev, 862 ii->intr_res, 863 INTR_TYPE_NET, 864 oce_fast_isr, NULL, ii, &ii->tag); 865 return rc; 866 867 } 868 869 void 870 oce_intr_free(POCE_SOFTC sc) 871 { 872 int i = 0; 873 874 for (i = 0; i < sc->intr_count; i++) { 875 876 if (sc->intrs[i].tag != NULL) 877 bus_teardown_intr(sc->dev, sc->intrs[i].intr_res, 878 sc->intrs[i].tag); 879 if (sc->intrs[i].tq != NULL) 880 taskqueue_free(sc->intrs[i].tq); 881 882 if (sc->intrs[i].intr_res != NULL) 883 bus_release_resource(sc->dev, SYS_RES_IRQ, 884 sc->intrs[i].irq_rr, 885 sc->intrs[i].intr_res); 886 sc->intrs[i].tag = NULL; 887 sc->intrs[i].intr_res = NULL; 888 } 889 890 if (sc->flags & OCE_FLAGS_USING_MSIX) 891 pci_release_msi(sc->dev); 892 893 } 894 895 /****************************************************************************** 896 * Media callbacks functions * 897 ******************************************************************************/ 898 899 static void 900 oce_media_status(if_t ifp, struct ifmediareq *req) 901 { 902 POCE_SOFTC sc = (POCE_SOFTC) if_getsoftc(ifp); 903 904 req->ifm_status = IFM_AVALID; 905 req->ifm_active = IFM_ETHER; 906 907 if (sc->link_status == 1) 908 req->ifm_status |= IFM_ACTIVE; 909 else 910 return; 911 912 switch (sc->link_speed) { 913 case 1: /* 10 Mbps */ 914 req->ifm_active |= IFM_10_T | IFM_FDX; 915 sc->speed = 10; 916 break; 917 case 2: /* 100 Mbps */ 918 req->ifm_active |= IFM_100_TX | IFM_FDX; 919 sc->speed = 100; 920 break; 921 case 3: /* 1 Gbps */ 922 req->ifm_active |= IFM_1000_T | IFM_FDX; 923 sc->speed = 1000; 924 break; 925 case 4: /* 10 Gbps */ 926 req->ifm_active |= IFM_10G_SR | IFM_FDX; 927 sc->speed = 10000; 928 break; 929 case 5: /* 20 Gbps */ 930 req->ifm_active |= IFM_10G_SR | IFM_FDX; 931 sc->speed = 20000; 932 break; 933 case 6: /* 25 Gbps */ 934 req->ifm_active |= IFM_10G_SR | IFM_FDX; 935 sc->speed = 25000; 936 break; 937 case 7: /* 40 Gbps */ 938 req->ifm_active |= IFM_40G_SR4 | IFM_FDX; 939 sc->speed = 40000; 940 break; 941 default: 942 sc->speed = 0; 943 break; 944 } 945 946 return; 947 } 948 949 int 950 oce_media_change(if_t ifp) 951 { 952 return 0; 953 } 954 955 static void oce_is_pkt_dest_bmc(POCE_SOFTC sc, 956 struct mbuf *m, boolean_t *os2bmc, 957 struct mbuf **m_new) 958 { 959 struct ether_header *eh = NULL; 960 961 eh = mtod(m, struct ether_header *); 962 963 if (!is_os2bmc_enabled(sc) || *os2bmc) { 964 *os2bmc = FALSE; 965 goto done; 966 } 967 if (!ETHER_IS_MULTICAST(eh->ether_dhost)) 968 goto done; 969 970 if (is_mc_allowed_on_bmc(sc, eh) || 971 is_bc_allowed_on_bmc(sc, eh) || 972 is_arp_allowed_on_bmc(sc, ntohs(eh->ether_type))) { 973 *os2bmc = TRUE; 974 goto done; 975 } 976 977 if (mtod(m, struct ip *)->ip_p == IPPROTO_IPV6) { 978 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); 979 uint8_t nexthdr = ip6->ip6_nxt; 980 if (nexthdr == IPPROTO_ICMPV6) { 981 struct icmp6_hdr *icmp6 = (struct icmp6_hdr *)(ip6 + 1); 982 switch (icmp6->icmp6_type) { 983 case ND_ROUTER_ADVERT: 984 *os2bmc = is_ipv6_ra_filt_enabled(sc); 985 goto done; 986 case ND_NEIGHBOR_ADVERT: 987 *os2bmc = is_ipv6_na_filt_enabled(sc); 988 goto done; 989 default: 990 break; 991 } 992 } 993 } 994 995 if (mtod(m, struct ip *)->ip_p == IPPROTO_UDP) { 996 struct ip *ip = mtod(m, struct ip *); 997 int iphlen = ip->ip_hl << 2; 998 struct udphdr *uh = (struct udphdr *)((caddr_t)ip + iphlen); 999 switch (uh->uh_dport) { 1000 case DHCP_CLIENT_PORT: 1001 *os2bmc = is_dhcp_client_filt_enabled(sc); 1002 goto done; 1003 case DHCP_SERVER_PORT: 1004 *os2bmc = is_dhcp_srvr_filt_enabled(sc); 1005 goto done; 1006 case NET_BIOS_PORT1: 1007 case NET_BIOS_PORT2: 1008 *os2bmc = is_nbios_filt_enabled(sc); 1009 goto done; 1010 case DHCPV6_RAS_PORT: 1011 *os2bmc = is_ipv6_ras_filt_enabled(sc); 1012 goto done; 1013 default: 1014 break; 1015 } 1016 } 1017 done: 1018 if (*os2bmc) { 1019 *m_new = m_dup(m, M_NOWAIT); 1020 if (!*m_new) { 1021 *os2bmc = FALSE; 1022 return; 1023 } 1024 *m_new = oce_insert_vlan_tag(sc, *m_new, NULL); 1025 } 1026 } 1027 1028 /***************************************************************************** 1029 * Transmit routines functions * 1030 *****************************************************************************/ 1031 1032 static int 1033 oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index) 1034 { 1035 int rc = 0, i, retry_cnt = 0; 1036 bus_dma_segment_t segs[OCE_MAX_TX_ELEMENTS]; 1037 struct mbuf *m, *m_temp, *m_new = NULL; 1038 struct oce_wq *wq = sc->wq[wq_index]; 1039 struct oce_packet_desc *pd; 1040 struct oce_nic_hdr_wqe *nichdr; 1041 struct oce_nic_frag_wqe *nicfrag; 1042 struct ether_header *eh = NULL; 1043 int num_wqes; 1044 uint32_t reg_value; 1045 boolean_t complete = TRUE; 1046 boolean_t os2bmc = FALSE; 1047 1048 m = *mpp; 1049 if (!m) 1050 return EINVAL; 1051 1052 if (!(m->m_flags & M_PKTHDR)) { 1053 rc = ENXIO; 1054 goto free_ret; 1055 } 1056 1057 /* Don't allow non-TSO packets longer than MTU */ 1058 if (!is_tso_pkt(m)) { 1059 eh = mtod(m, struct ether_header *); 1060 if(m->m_pkthdr.len > ETHER_MAX_FRAME(sc->ifp, eh->ether_type, FALSE)) 1061 goto free_ret; 1062 } 1063 1064 if(oce_tx_asic_stall_verify(sc, m)) { 1065 m = oce_insert_vlan_tag(sc, m, &complete); 1066 if(!m) { 1067 device_printf(sc->dev, "Insertion unsuccessful\n"); 1068 return 0; 1069 } 1070 } 1071 1072 /* Lancer, SH ASIC has a bug wherein Packets that are 32 bytes or less 1073 * may cause a transmit stall on that port. So the work-around is to 1074 * pad short packets (<= 32 bytes) to a 36-byte length. 1075 */ 1076 if(IS_SH(sc) || IS_XE201(sc) ) { 1077 if(m->m_pkthdr.len <= 32) { 1078 char buf[36]; 1079 bzero((void *)buf, 36); 1080 m_append(m, (36 - m->m_pkthdr.len), buf); 1081 } 1082 } 1083 1084 tx_start: 1085 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1086 /* consolidate packet buffers for TSO/LSO segment offload */ 1087 #if defined(INET6) || defined(INET) 1088 m = oce_tso_setup(sc, mpp); 1089 #else 1090 m = NULL; 1091 #endif 1092 if (m == NULL) { 1093 rc = ENXIO; 1094 goto free_ret; 1095 } 1096 } 1097 1098 pd = &wq->pckts[wq->pkt_desc_head]; 1099 1100 retry: 1101 rc = bus_dmamap_load_mbuf_sg(wq->tag, 1102 pd->map, 1103 m, segs, &pd->nsegs, BUS_DMA_NOWAIT); 1104 if (rc == 0) { 1105 num_wqes = pd->nsegs + 1; 1106 if (IS_BE(sc) || IS_SH(sc)) { 1107 /*Dummy required only for BE3.*/ 1108 if (num_wqes & 1) 1109 num_wqes++; 1110 } 1111 if (num_wqes >= RING_NUM_FREE(wq->ring)) { 1112 bus_dmamap_unload(wq->tag, pd->map); 1113 return EBUSY; 1114 } 1115 atomic_store_rel_int(&wq->pkt_desc_head, 1116 (wq->pkt_desc_head + 1) % \ 1117 OCE_WQ_PACKET_ARRAY_SIZE); 1118 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_PREWRITE); 1119 pd->mbuf = m; 1120 1121 nichdr = 1122 RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_hdr_wqe); 1123 nichdr->u0.dw[0] = 0; 1124 nichdr->u0.dw[1] = 0; 1125 nichdr->u0.dw[2] = 0; 1126 nichdr->u0.dw[3] = 0; 1127 1128 nichdr->u0.s.complete = complete; 1129 nichdr->u0.s.mgmt = os2bmc; 1130 nichdr->u0.s.event = 1; 1131 nichdr->u0.s.crc = 1; 1132 nichdr->u0.s.forward = 0; 1133 nichdr->u0.s.ipcs = (m->m_pkthdr.csum_flags & CSUM_IP) ? 1 : 0; 1134 nichdr->u0.s.udpcs = 1135 (m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0; 1136 nichdr->u0.s.tcpcs = 1137 (m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0; 1138 nichdr->u0.s.num_wqe = num_wqes; 1139 nichdr->u0.s.total_length = m->m_pkthdr.len; 1140 1141 if (m->m_flags & M_VLANTAG) { 1142 nichdr->u0.s.vlan = 1; /*Vlan present*/ 1143 nichdr->u0.s.vlan_tag = m->m_pkthdr.ether_vtag; 1144 } 1145 1146 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1147 if (m->m_pkthdr.tso_segsz) { 1148 nichdr->u0.s.lso = 1; 1149 nichdr->u0.s.lso_mss = m->m_pkthdr.tso_segsz; 1150 } 1151 if (!IS_BE(sc) || !IS_SH(sc)) 1152 nichdr->u0.s.ipcs = 1; 1153 } 1154 1155 RING_PUT(wq->ring, 1); 1156 atomic_add_int(&wq->ring->num_used, 1); 1157 1158 for (i = 0; i < pd->nsegs; i++) { 1159 nicfrag = 1160 RING_GET_PRODUCER_ITEM_VA(wq->ring, 1161 struct oce_nic_frag_wqe); 1162 nicfrag->u0.s.rsvd0 = 0; 1163 nicfrag->u0.s.frag_pa_hi = ADDR_HI(segs[i].ds_addr); 1164 nicfrag->u0.s.frag_pa_lo = ADDR_LO(segs[i].ds_addr); 1165 nicfrag->u0.s.frag_len = segs[i].ds_len; 1166 pd->wqe_idx = wq->ring->pidx; 1167 RING_PUT(wq->ring, 1); 1168 atomic_add_int(&wq->ring->num_used, 1); 1169 } 1170 if (num_wqes > (pd->nsegs + 1)) { 1171 nicfrag = 1172 RING_GET_PRODUCER_ITEM_VA(wq->ring, 1173 struct oce_nic_frag_wqe); 1174 nicfrag->u0.dw[0] = 0; 1175 nicfrag->u0.dw[1] = 0; 1176 nicfrag->u0.dw[2] = 0; 1177 nicfrag->u0.dw[3] = 0; 1178 pd->wqe_idx = wq->ring->pidx; 1179 RING_PUT(wq->ring, 1); 1180 atomic_add_int(&wq->ring->num_used, 1); 1181 pd->nsegs++; 1182 } 1183 1184 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1); 1185 wq->tx_stats.tx_reqs++; 1186 wq->tx_stats.tx_wrbs += num_wqes; 1187 wq->tx_stats.tx_bytes += m->m_pkthdr.len; 1188 wq->tx_stats.tx_pkts++; 1189 1190 bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map, 1191 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1192 reg_value = (num_wqes << 16) | wq->wq_id; 1193 1194 /* if os2bmc is not enabled or if the pkt is already tagged as 1195 bmc, do nothing 1196 */ 1197 oce_is_pkt_dest_bmc(sc, m, &os2bmc, &m_new); 1198 1199 if_inc_counter(sc->ifp, IFCOUNTER_OBYTES, m->m_pkthdr.len); 1200 if (m->m_flags & M_MCAST) 1201 if_inc_counter(sc->ifp, IFCOUNTER_OMCASTS, 1); 1202 ETHER_BPF_MTAP(sc->ifp, m); 1203 1204 OCE_WRITE_REG32(sc, db, wq->db_offset, reg_value); 1205 1206 } else if (rc == EFBIG) { 1207 if (retry_cnt == 0) { 1208 m_temp = m_defrag(m, M_NOWAIT); 1209 if (m_temp == NULL) 1210 goto free_ret; 1211 m = m_temp; 1212 *mpp = m_temp; 1213 retry_cnt = retry_cnt + 1; 1214 goto retry; 1215 } else 1216 goto free_ret; 1217 } else if (rc == ENOMEM) 1218 return rc; 1219 else 1220 goto free_ret; 1221 1222 if (os2bmc) { 1223 m = m_new; 1224 goto tx_start; 1225 } 1226 1227 return 0; 1228 1229 free_ret: 1230 m_freem(*mpp); 1231 *mpp = NULL; 1232 return rc; 1233 } 1234 1235 static void 1236 oce_process_tx_completion(struct oce_wq *wq) 1237 { 1238 struct oce_packet_desc *pd; 1239 POCE_SOFTC sc = (POCE_SOFTC) wq->parent; 1240 struct mbuf *m; 1241 1242 pd = &wq->pckts[wq->pkt_desc_tail]; 1243 atomic_store_rel_int(&wq->pkt_desc_tail, 1244 (wq->pkt_desc_tail + 1) % OCE_WQ_PACKET_ARRAY_SIZE); 1245 atomic_subtract_int(&wq->ring->num_used, pd->nsegs + 1); 1246 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1247 bus_dmamap_unload(wq->tag, pd->map); 1248 1249 m = pd->mbuf; 1250 m_freem(m); 1251 pd->mbuf = NULL; 1252 1253 if (if_getdrvflags(sc->ifp) & IFF_DRV_OACTIVE) { 1254 if (wq->ring->num_used < (wq->ring->num_items / 2)) { 1255 if_setdrvflagbits(sc->ifp, 0, (IFF_DRV_OACTIVE)); 1256 oce_tx_restart(sc, wq); 1257 } 1258 } 1259 } 1260 1261 static void 1262 oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq) 1263 { 1264 1265 if ((if_getdrvflags(sc->ifp) & IFF_DRV_RUNNING) != IFF_DRV_RUNNING) 1266 return; 1267 1268 if (!drbr_empty(sc->ifp, wq->br)) 1269 taskqueue_enqueue(taskqueue_swi, &wq->txtask); 1270 1271 } 1272 1273 #if defined(INET6) || defined(INET) 1274 static struct mbuf * 1275 oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp) 1276 { 1277 struct mbuf *m; 1278 #ifdef INET 1279 struct ip *ip; 1280 #endif 1281 #ifdef INET6 1282 struct ip6_hdr *ip6; 1283 #endif 1284 struct ether_vlan_header *eh; 1285 struct tcphdr *th; 1286 uint16_t etype; 1287 int total_len = 0, ehdrlen = 0; 1288 1289 m = *mpp; 1290 1291 if (M_WRITABLE(m) == 0) { 1292 m = m_dup(*mpp, M_NOWAIT); 1293 if (!m) 1294 return NULL; 1295 m_freem(*mpp); 1296 *mpp = m; 1297 } 1298 1299 eh = mtod(m, struct ether_vlan_header *); 1300 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 1301 etype = ntohs(eh->evl_proto); 1302 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 1303 } else { 1304 etype = ntohs(eh->evl_encap_proto); 1305 ehdrlen = ETHER_HDR_LEN; 1306 } 1307 1308 switch (etype) { 1309 #ifdef INET 1310 case ETHERTYPE_IP: 1311 ip = (struct ip *)(m->m_data + ehdrlen); 1312 if (ip->ip_p != IPPROTO_TCP) 1313 return NULL; 1314 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2)); 1315 1316 total_len = ehdrlen + (ip->ip_hl << 2) + (th->th_off << 2); 1317 break; 1318 #endif 1319 #ifdef INET6 1320 case ETHERTYPE_IPV6: 1321 ip6 = (struct ip6_hdr *)(m->m_data + ehdrlen); 1322 if (ip6->ip6_nxt != IPPROTO_TCP) 1323 return NULL; 1324 th = (struct tcphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr)); 1325 1326 total_len = ehdrlen + sizeof(struct ip6_hdr) + (th->th_off << 2); 1327 break; 1328 #endif 1329 default: 1330 return NULL; 1331 } 1332 1333 m = m_pullup(m, total_len); 1334 *mpp = m; 1335 return m; 1336 } 1337 #endif /* INET6 || INET */ 1338 1339 void 1340 oce_tx_task(void *arg, int npending) 1341 { 1342 struct oce_wq *wq = arg; 1343 POCE_SOFTC sc = wq->parent; 1344 if_t ifp = sc->ifp; 1345 int rc = 0; 1346 1347 LOCK(&wq->tx_lock); 1348 rc = oce_multiq_transmit(ifp, NULL, wq); 1349 if (rc) { 1350 device_printf(sc->dev, 1351 "TX[%d] restart failed\n", wq->queue_index); 1352 } 1353 UNLOCK(&wq->tx_lock); 1354 } 1355 1356 void 1357 oce_start(if_t ifp) 1358 { 1359 POCE_SOFTC sc = if_getsoftc(ifp); 1360 struct mbuf *m; 1361 int rc = 0; 1362 int def_q = 0; /* Defualt tx queue is 0*/ 1363 1364 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 1365 IFF_DRV_RUNNING) 1366 return; 1367 1368 if (!sc->link_status) 1369 return; 1370 1371 while (true) { 1372 m = if_dequeue(sc->ifp); 1373 if (m == NULL) 1374 break; 1375 1376 LOCK(&sc->wq[def_q]->tx_lock); 1377 rc = oce_tx(sc, &m, def_q); 1378 UNLOCK(&sc->wq[def_q]->tx_lock); 1379 if (rc) { 1380 if (m != NULL) { 1381 sc->wq[def_q]->tx_stats.tx_stops ++; 1382 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0); 1383 if_sendq_prepend(ifp, m); 1384 m = NULL; 1385 } 1386 break; 1387 } 1388 } 1389 } 1390 1391 /* Handle the Completion Queue for transmit */ 1392 uint16_t 1393 oce_wq_handler(void *arg) 1394 { 1395 struct oce_wq *wq = (struct oce_wq *)arg; 1396 POCE_SOFTC sc = wq->parent; 1397 struct oce_cq *cq = wq->cq; 1398 struct oce_nic_tx_cqe *cqe; 1399 int num_cqes = 0; 1400 1401 LOCK(&wq->tx_compl_lock); 1402 bus_dmamap_sync(cq->ring->dma.tag, 1403 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1404 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 1405 while (cqe->u0.dw[3]) { 1406 DW_SWAP((uint32_t *) cqe, sizeof(oce_wq_cqe)); 1407 1408 wq->ring->cidx = cqe->u0.s.wqe_index + 1; 1409 if (wq->ring->cidx >= wq->ring->num_items) 1410 wq->ring->cidx -= wq->ring->num_items; 1411 1412 oce_process_tx_completion(wq); 1413 wq->tx_stats.tx_compl++; 1414 cqe->u0.dw[3] = 0; 1415 RING_GET(cq->ring, 1); 1416 bus_dmamap_sync(cq->ring->dma.tag, 1417 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1418 cqe = 1419 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 1420 num_cqes++; 1421 } 1422 1423 if (num_cqes) 1424 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); 1425 1426 UNLOCK(&wq->tx_compl_lock); 1427 return num_cqes; 1428 } 1429 1430 static int 1431 oce_multiq_transmit(if_t ifp, struct mbuf *m, struct oce_wq *wq) 1432 { 1433 POCE_SOFTC sc = if_getsoftc(ifp); 1434 int status = 0, queue_index = 0; 1435 struct mbuf *next = NULL; 1436 struct buf_ring *br = NULL; 1437 1438 br = wq->br; 1439 queue_index = wq->queue_index; 1440 1441 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 1442 IFF_DRV_RUNNING) { 1443 if (m != NULL) 1444 status = drbr_enqueue(ifp, br, m); 1445 return status; 1446 } 1447 1448 if (m != NULL) { 1449 if ((status = drbr_enqueue(ifp, br, m)) != 0) 1450 return status; 1451 } 1452 while ((next = drbr_peek(ifp, br)) != NULL) { 1453 if (oce_tx(sc, &next, queue_index)) { 1454 if (next == NULL) { 1455 drbr_advance(ifp, br); 1456 } else { 1457 drbr_putback(ifp, br, next); 1458 wq->tx_stats.tx_stops ++; 1459 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0); 1460 } 1461 break; 1462 } 1463 drbr_advance(ifp, br); 1464 } 1465 1466 return 0; 1467 } 1468 1469 /***************************************************************************** 1470 * Receive routines functions * 1471 *****************************************************************************/ 1472 1473 static void 1474 oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2) 1475 { 1476 uint32_t *p; 1477 struct ether_header *eh = NULL; 1478 struct tcphdr *tcp_hdr = NULL; 1479 struct ip *ip4_hdr = NULL; 1480 struct ip6_hdr *ip6 = NULL; 1481 uint32_t payload_len = 0; 1482 1483 eh = mtod(m, struct ether_header *); 1484 /* correct IP header */ 1485 if(!cqe2->ipv6_frame) { 1486 ip4_hdr = (struct ip *)((char*)eh + sizeof(struct ether_header)); 1487 ip4_hdr->ip_ttl = cqe2->frame_lifespan; 1488 ip4_hdr->ip_len = htons(cqe2->coalesced_size - sizeof(struct ether_header)); 1489 tcp_hdr = (struct tcphdr *)((char*)ip4_hdr + sizeof(struct ip)); 1490 }else { 1491 ip6 = (struct ip6_hdr *)((char*)eh + sizeof(struct ether_header)); 1492 ip6->ip6_ctlun.ip6_un1.ip6_un1_hlim = cqe2->frame_lifespan; 1493 payload_len = cqe2->coalesced_size - sizeof(struct ether_header) 1494 - sizeof(struct ip6_hdr); 1495 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = htons(payload_len); 1496 tcp_hdr = (struct tcphdr *)((char*)ip6 + sizeof(struct ip6_hdr)); 1497 } 1498 1499 /* correct tcp header */ 1500 tcp_hdr->th_ack = htonl(cqe2->tcp_ack_num); 1501 if(cqe2->push) { 1502 tcp_hdr->th_flags |= TH_PUSH; 1503 } 1504 tcp_hdr->th_win = htons(cqe2->tcp_window); 1505 tcp_hdr->th_sum = 0xffff; 1506 if(cqe2->ts_opt) { 1507 p = (uint32_t *)((char*)tcp_hdr + sizeof(struct tcphdr) + 2); 1508 *p = cqe1->tcp_timestamp_val; 1509 *(p+1) = cqe1->tcp_timestamp_ecr; 1510 } 1511 1512 return; 1513 } 1514 1515 static void 1516 oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m) 1517 { 1518 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1519 uint32_t i = 0, frag_len = 0; 1520 uint32_t len = cqe_info->pkt_size; 1521 struct oce_packet_desc *pd; 1522 struct mbuf *tail = NULL; 1523 1524 for (i = 0; i < cqe_info->num_frags; i++) { 1525 if (rq->ring->cidx == rq->ring->pidx) { 1526 device_printf(sc->dev, 1527 "oce_rx_mbuf_chain: Invalid RX completion - Queue is empty\n"); 1528 return; 1529 } 1530 pd = &rq->pckts[rq->ring->cidx]; 1531 1532 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1533 bus_dmamap_unload(rq->tag, pd->map); 1534 RING_GET(rq->ring, 1); 1535 rq->pending--; 1536 1537 frag_len = (len > rq->cfg.frag_size) ? rq->cfg.frag_size : len; 1538 pd->mbuf->m_len = frag_len; 1539 1540 if (tail != NULL) { 1541 /* additional fragments */ 1542 pd->mbuf->m_flags &= ~M_PKTHDR; 1543 tail->m_next = pd->mbuf; 1544 if(rq->islro) 1545 tail->m_nextpkt = NULL; 1546 tail = pd->mbuf; 1547 } else { 1548 /* first fragment, fill out much of the packet header */ 1549 pd->mbuf->m_pkthdr.len = len; 1550 if(rq->islro) 1551 pd->mbuf->m_nextpkt = NULL; 1552 pd->mbuf->m_pkthdr.csum_flags = 0; 1553 if (IF_CSUM_ENABLED(sc)) { 1554 if (cqe_info->l4_cksum_pass) { 1555 if(!cqe_info->ipv6_frame) { /* IPV4 */ 1556 pd->mbuf->m_pkthdr.csum_flags |= 1557 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1558 }else { /* IPV6 frame */ 1559 if(rq->islro) { 1560 pd->mbuf->m_pkthdr.csum_flags |= 1561 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1562 } 1563 } 1564 pd->mbuf->m_pkthdr.csum_data = 0xffff; 1565 } 1566 if (cqe_info->ip_cksum_pass) { 1567 pd->mbuf->m_pkthdr.csum_flags |= 1568 (CSUM_IP_CHECKED|CSUM_IP_VALID); 1569 } 1570 } 1571 *m = tail = pd->mbuf; 1572 } 1573 pd->mbuf = NULL; 1574 len -= frag_len; 1575 } 1576 1577 return; 1578 } 1579 1580 static void 1581 oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2) 1582 { 1583 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1584 struct nic_hwlro_cqe_part1 *cqe1 = NULL; 1585 struct mbuf *m = NULL; 1586 struct oce_common_cqe_info cq_info; 1587 1588 /* parse cqe */ 1589 if(cqe2 == NULL) { 1590 cq_info.pkt_size = cqe->pkt_size; 1591 cq_info.vtag = cqe->vlan_tag; 1592 cq_info.l4_cksum_pass = cqe->l4_cksum_pass; 1593 cq_info.ip_cksum_pass = cqe->ip_cksum_pass; 1594 cq_info.ipv6_frame = cqe->ipv6_frame; 1595 cq_info.vtp = cqe->vtp; 1596 cq_info.qnq = cqe->qnq; 1597 }else { 1598 cqe1 = (struct nic_hwlro_cqe_part1 *)cqe; 1599 cq_info.pkt_size = cqe2->coalesced_size; 1600 cq_info.vtag = cqe2->vlan_tag; 1601 cq_info.l4_cksum_pass = cqe2->l4_cksum_pass; 1602 cq_info.ip_cksum_pass = cqe2->ip_cksum_pass; 1603 cq_info.ipv6_frame = cqe2->ipv6_frame; 1604 cq_info.vtp = cqe2->vtp; 1605 cq_info.qnq = cqe1->qnq; 1606 } 1607 1608 cq_info.vtag = BSWAP_16(cq_info.vtag); 1609 1610 cq_info.num_frags = cq_info.pkt_size / rq->cfg.frag_size; 1611 if(cq_info.pkt_size % rq->cfg.frag_size) 1612 cq_info.num_frags++; 1613 1614 oce_rx_mbuf_chain(rq, &cq_info, &m); 1615 1616 if (m) { 1617 if(cqe2) { 1618 //assert(cqe2->valid != 0); 1619 1620 //assert(cqe2->cqe_type != 2); 1621 oce_correct_header(m, cqe1, cqe2); 1622 } 1623 1624 m->m_pkthdr.rcvif = sc->ifp; 1625 if (rq->queue_index) 1626 m->m_pkthdr.flowid = (rq->queue_index - 1); 1627 else 1628 m->m_pkthdr.flowid = rq->queue_index; 1629 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 1630 1631 /* This deternies if vlan tag is Valid */ 1632 if (cq_info.vtp) { 1633 if (sc->function_mode & FNM_FLEX10_MODE) { 1634 /* FLEX10. If QnQ is not set, neglect VLAN */ 1635 if (cq_info.qnq) { 1636 m->m_pkthdr.ether_vtag = cq_info.vtag; 1637 m->m_flags |= M_VLANTAG; 1638 } 1639 } else if (sc->pvid != (cq_info.vtag & VLAN_VID_MASK)) { 1640 /* In UMC mode generally pvid will be striped by 1641 hw. But in some cases we have seen it comes 1642 with pvid. So if pvid == vlan, neglect vlan. 1643 */ 1644 m->m_pkthdr.ether_vtag = cq_info.vtag; 1645 m->m_flags |= M_VLANTAG; 1646 } 1647 } 1648 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 1649 1650 if_input(sc->ifp, m); 1651 1652 /* Update rx stats per queue */ 1653 rq->rx_stats.rx_pkts++; 1654 rq->rx_stats.rx_bytes += cq_info.pkt_size; 1655 rq->rx_stats.rx_frags += cq_info.num_frags; 1656 rq->rx_stats.rx_ucast_pkts++; 1657 } 1658 return; 1659 } 1660 1661 static void 1662 oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe) 1663 { 1664 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1665 int len; 1666 struct mbuf *m = NULL; 1667 struct oce_common_cqe_info cq_info; 1668 uint16_t vtag = 0; 1669 1670 /* Is it a flush compl that has no data */ 1671 if(!cqe->u0.s.num_fragments) 1672 goto exit; 1673 1674 len = cqe->u0.s.pkt_size; 1675 if (!len) { 1676 /*partial DMA workaround for Lancer*/ 1677 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); 1678 goto exit; 1679 } 1680 1681 if (!oce_cqe_portid_valid(sc, cqe)) { 1682 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); 1683 goto exit; 1684 } 1685 1686 /* Get vlan_tag value */ 1687 if(IS_BE(sc) || IS_SH(sc)) 1688 vtag = BSWAP_16(cqe->u0.s.vlan_tag); 1689 else 1690 vtag = cqe->u0.s.vlan_tag; 1691 1692 cq_info.l4_cksum_pass = cqe->u0.s.l4_cksum_pass; 1693 cq_info.ip_cksum_pass = cqe->u0.s.ip_cksum_pass; 1694 cq_info.ipv6_frame = cqe->u0.s.ip_ver; 1695 cq_info.num_frags = cqe->u0.s.num_fragments; 1696 cq_info.pkt_size = cqe->u0.s.pkt_size; 1697 1698 oce_rx_mbuf_chain(rq, &cq_info, &m); 1699 1700 if (m) { 1701 m->m_pkthdr.rcvif = sc->ifp; 1702 if (rq->queue_index) 1703 m->m_pkthdr.flowid = (rq->queue_index - 1); 1704 else 1705 m->m_pkthdr.flowid = rq->queue_index; 1706 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 1707 1708 /* This deternies if vlan tag is Valid */ 1709 if (oce_cqe_vtp_valid(sc, cqe)) { 1710 if (sc->function_mode & FNM_FLEX10_MODE) { 1711 /* FLEX10. If QnQ is not set, neglect VLAN */ 1712 if (cqe->u0.s.qnq) { 1713 m->m_pkthdr.ether_vtag = vtag; 1714 m->m_flags |= M_VLANTAG; 1715 } 1716 } else if (sc->pvid != (vtag & VLAN_VID_MASK)) { 1717 /* In UMC mode generally pvid will be striped by 1718 hw. But in some cases we have seen it comes 1719 with pvid. So if pvid == vlan, neglect vlan. 1720 */ 1721 m->m_pkthdr.ether_vtag = vtag; 1722 m->m_flags |= M_VLANTAG; 1723 } 1724 } 1725 1726 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 1727 #if defined(INET6) || defined(INET) 1728 /* Try to queue to LRO */ 1729 if (IF_LRO_ENABLED(sc) && 1730 (cqe->u0.s.ip_cksum_pass) && 1731 (cqe->u0.s.l4_cksum_pass) && 1732 (!cqe->u0.s.ip_ver) && 1733 (rq->lro.lro_cnt != 0)) { 1734 if (tcp_lro_rx(&rq->lro, m, 0) == 0) { 1735 rq->lro_pkts_queued ++; 1736 goto post_done; 1737 } 1738 /* If LRO posting fails then try to post to STACK */ 1739 } 1740 #endif 1741 1742 if_input(sc->ifp, m); 1743 #if defined(INET6) || defined(INET) 1744 post_done: 1745 #endif 1746 /* Update rx stats per queue */ 1747 rq->rx_stats.rx_pkts++; 1748 rq->rx_stats.rx_bytes += cqe->u0.s.pkt_size; 1749 rq->rx_stats.rx_frags += cqe->u0.s.num_fragments; 1750 if (cqe->u0.s.pkt_type == OCE_MULTICAST_PACKET) 1751 rq->rx_stats.rx_mcast_pkts++; 1752 if (cqe->u0.s.pkt_type == OCE_UNICAST_PACKET) 1753 rq->rx_stats.rx_ucast_pkts++; 1754 } 1755 exit: 1756 return; 1757 } 1758 1759 void 1760 oce_discard_rx_comp(struct oce_rq *rq, int num_frags) 1761 { 1762 uint32_t i = 0; 1763 struct oce_packet_desc *pd; 1764 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1765 1766 for (i = 0; i < num_frags; i++) { 1767 if (rq->ring->cidx == rq->ring->pidx) { 1768 device_printf(sc->dev, 1769 "oce_discard_rx_comp: Invalid RX completion - Queue is empty\n"); 1770 return; 1771 } 1772 pd = &rq->pckts[rq->ring->cidx]; 1773 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1774 bus_dmamap_unload(rq->tag, pd->map); 1775 if (pd->mbuf != NULL) { 1776 m_freem(pd->mbuf); 1777 pd->mbuf = NULL; 1778 } 1779 1780 RING_GET(rq->ring, 1); 1781 rq->pending--; 1782 } 1783 } 1784 1785 static int 1786 oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe) 1787 { 1788 struct oce_nic_rx_cqe_v1 *cqe_v1; 1789 int vtp = 0; 1790 1791 if (sc->be3_native) { 1792 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe; 1793 vtp = cqe_v1->u0.s.vlan_tag_present; 1794 } else 1795 vtp = cqe->u0.s.vlan_tag_present; 1796 1797 return vtp; 1798 1799 } 1800 1801 static int 1802 oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe) 1803 { 1804 struct oce_nic_rx_cqe_v1 *cqe_v1; 1805 int port_id = 0; 1806 1807 if (sc->be3_native && (IS_BE(sc) || IS_SH(sc))) { 1808 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe; 1809 port_id = cqe_v1->u0.s.port; 1810 if (sc->port_id != port_id) 1811 return 0; 1812 } else 1813 ;/* For BE3 legacy and Lancer this is dummy */ 1814 1815 return 1; 1816 1817 } 1818 1819 #if defined(INET6) || defined(INET) 1820 void 1821 oce_rx_flush_lro(struct oce_rq *rq) 1822 { 1823 struct lro_ctrl *lro = &rq->lro; 1824 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1825 1826 if (!IF_LRO_ENABLED(sc)) 1827 return; 1828 1829 tcp_lro_flush_all(lro); 1830 rq->lro_pkts_queued = 0; 1831 1832 return; 1833 } 1834 1835 static int 1836 oce_init_lro(POCE_SOFTC sc) 1837 { 1838 struct lro_ctrl *lro = NULL; 1839 int i = 0, rc = 0; 1840 1841 for (i = 0; i < sc->nrqs; i++) { 1842 lro = &sc->rq[i]->lro; 1843 rc = tcp_lro_init(lro); 1844 if (rc != 0) { 1845 device_printf(sc->dev, "LRO init failed\n"); 1846 return rc; 1847 } 1848 lro->ifp = sc->ifp; 1849 } 1850 1851 return rc; 1852 } 1853 1854 void 1855 oce_free_lro(POCE_SOFTC sc) 1856 { 1857 struct lro_ctrl *lro = NULL; 1858 int i = 0; 1859 1860 for (i = 0; i < sc->nrqs; i++) { 1861 lro = &sc->rq[i]->lro; 1862 if (lro) 1863 tcp_lro_free(lro); 1864 } 1865 } 1866 #endif 1867 1868 int 1869 oce_alloc_rx_bufs(struct oce_rq *rq, int count) 1870 { 1871 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1872 int i, rc; 1873 struct oce_packet_desc *pd; 1874 bus_dma_segment_t segs[6]; 1875 int nsegs, added = 0; 1876 struct oce_nic_rqe *rqe; 1877 pd_rxulp_db_t rxdb_reg; 1878 uint32_t val = 0; 1879 uint32_t oce_max_rq_posts = 64; 1880 1881 bzero(&rxdb_reg, sizeof(pd_rxulp_db_t)); 1882 for (i = 0; i < count; i++) { 1883 pd = &rq->pckts[rq->ring->pidx]; 1884 pd->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, oce_rq_buf_size); 1885 if (pd->mbuf == NULL) { 1886 device_printf(sc->dev, "mbuf allocation failed, size = %d\n",oce_rq_buf_size); 1887 break; 1888 } 1889 pd->mbuf->m_nextpkt = NULL; 1890 1891 pd->mbuf->m_len = pd->mbuf->m_pkthdr.len = rq->cfg.frag_size; 1892 1893 rc = bus_dmamap_load_mbuf_sg(rq->tag, 1894 pd->map, 1895 pd->mbuf, 1896 segs, &nsegs, BUS_DMA_NOWAIT); 1897 if (rc) { 1898 m_free(pd->mbuf); 1899 device_printf(sc->dev, "bus_dmamap_load_mbuf_sg failed rc = %d\n", rc); 1900 break; 1901 } 1902 1903 if (nsegs != 1) { 1904 i--; 1905 continue; 1906 } 1907 1908 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_PREREAD); 1909 1910 rqe = RING_GET_PRODUCER_ITEM_VA(rq->ring, struct oce_nic_rqe); 1911 rqe->u0.s.frag_pa_hi = ADDR_HI(segs[0].ds_addr); 1912 rqe->u0.s.frag_pa_lo = ADDR_LO(segs[0].ds_addr); 1913 DW_SWAP(u32ptr(rqe), sizeof(struct oce_nic_rqe)); 1914 RING_PUT(rq->ring, 1); 1915 added++; 1916 rq->pending++; 1917 } 1918 oce_max_rq_posts = sc->enable_hwlro ? OCE_HWLRO_MAX_RQ_POSTS : OCE_MAX_RQ_POSTS; 1919 if (added != 0) { 1920 for (i = added / oce_max_rq_posts; i > 0; i--) { 1921 rxdb_reg.bits.num_posted = oce_max_rq_posts; 1922 rxdb_reg.bits.qid = rq->rq_id; 1923 if(rq->islro) { 1924 val |= rq->rq_id & DB_LRO_RQ_ID_MASK; 1925 val |= oce_max_rq_posts << 16; 1926 OCE_WRITE_REG32(sc, db, DB_OFFSET, val); 1927 }else { 1928 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); 1929 } 1930 added -= oce_max_rq_posts; 1931 } 1932 if (added > 0) { 1933 rxdb_reg.bits.qid = rq->rq_id; 1934 rxdb_reg.bits.num_posted = added; 1935 if(rq->islro) { 1936 val |= rq->rq_id & DB_LRO_RQ_ID_MASK; 1937 val |= added << 16; 1938 OCE_WRITE_REG32(sc, db, DB_OFFSET, val); 1939 }else { 1940 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); 1941 } 1942 } 1943 } 1944 1945 return 0; 1946 } 1947 1948 static void 1949 oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq) 1950 { 1951 if (num_cqes) { 1952 oce_arm_cq(sc, rq->cq->cq_id, num_cqes, FALSE); 1953 if(!sc->enable_hwlro) { 1954 if((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) > 1) 1955 oce_alloc_rx_bufs(rq, ((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) - 1)); 1956 }else { 1957 if ((OCE_RQ_PACKET_ARRAY_SIZE -1 - rq->pending) > 64) 1958 oce_alloc_rx_bufs(rq, 64); 1959 } 1960 } 1961 1962 return; 1963 } 1964 1965 uint16_t 1966 oce_rq_handler_lro(void *arg) 1967 { 1968 struct oce_rq *rq = (struct oce_rq *)arg; 1969 struct oce_cq *cq = rq->cq; 1970 POCE_SOFTC sc = rq->parent; 1971 struct nic_hwlro_singleton_cqe *cqe; 1972 struct nic_hwlro_cqe_part2 *cqe2; 1973 int num_cqes = 0; 1974 1975 LOCK(&rq->rx_lock); 1976 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1977 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); 1978 while (cqe->valid) { 1979 if(cqe->cqe_type == 0) { /* singleton cqe */ 1980 /* we should not get singleton cqe after cqe1 on same rq */ 1981 if(rq->cqe_firstpart != NULL) { 1982 device_printf(sc->dev, "Got singleton cqe after cqe1 \n"); 1983 goto exit_rq_handler_lro; 1984 } 1985 if(cqe->error != 0) { 1986 rq->rx_stats.rxcp_err++; 1987 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 1988 } 1989 oce_rx_lro(rq, cqe, NULL); 1990 rq->rx_stats.rx_compl++; 1991 cqe->valid = 0; 1992 RING_GET(cq->ring, 1); 1993 num_cqes++; 1994 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 1995 break; 1996 }else if(cqe->cqe_type == 0x1) { /* first part */ 1997 /* we should not get cqe1 after cqe1 on same rq */ 1998 if(rq->cqe_firstpart != NULL) { 1999 device_printf(sc->dev, "Got cqe1 after cqe1 \n"); 2000 goto exit_rq_handler_lro; 2001 } 2002 rq->cqe_firstpart = (struct nic_hwlro_cqe_part1 *)cqe; 2003 RING_GET(cq->ring, 1); 2004 }else if(cqe->cqe_type == 0x2) { /* second part */ 2005 cqe2 = (struct nic_hwlro_cqe_part2 *)cqe; 2006 if(cqe2->error != 0) { 2007 rq->rx_stats.rxcp_err++; 2008 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2009 } 2010 /* We should not get cqe2 without cqe1 */ 2011 if(rq->cqe_firstpart == NULL) { 2012 device_printf(sc->dev, "Got cqe2 without cqe1 \n"); 2013 goto exit_rq_handler_lro; 2014 } 2015 oce_rx_lro(rq, (struct nic_hwlro_singleton_cqe *)rq->cqe_firstpart, cqe2); 2016 2017 rq->rx_stats.rx_compl++; 2018 rq->cqe_firstpart->valid = 0; 2019 cqe2->valid = 0; 2020 rq->cqe_firstpart = NULL; 2021 2022 RING_GET(cq->ring, 1); 2023 num_cqes += 2; 2024 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2025 break; 2026 } 2027 2028 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2029 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); 2030 } 2031 oce_check_rx_bufs(sc, num_cqes, rq); 2032 exit_rq_handler_lro: 2033 UNLOCK(&rq->rx_lock); 2034 return 0; 2035 } 2036 2037 /* Handle the Completion Queue for receive */ 2038 uint16_t 2039 oce_rq_handler(void *arg) 2040 { 2041 struct epoch_tracker et; 2042 struct oce_rq *rq = (struct oce_rq *)arg; 2043 struct oce_cq *cq = rq->cq; 2044 POCE_SOFTC sc = rq->parent; 2045 struct oce_nic_rx_cqe *cqe; 2046 int num_cqes = 0; 2047 2048 NET_EPOCH_ENTER(et); 2049 if(rq->islro) { 2050 oce_rq_handler_lro(arg); 2051 NET_EPOCH_EXIT(et); 2052 return 0; 2053 } 2054 LOCK(&rq->rx_lock); 2055 bus_dmamap_sync(cq->ring->dma.tag, 2056 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2057 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); 2058 while (cqe->u0.dw[2]) { 2059 DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe)); 2060 2061 if (cqe->u0.s.error == 0) { 2062 oce_rx(rq, cqe); 2063 } else { 2064 rq->rx_stats.rxcp_err++; 2065 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2066 /* Post L3/L4 errors to stack.*/ 2067 oce_rx(rq, cqe); 2068 } 2069 rq->rx_stats.rx_compl++; 2070 cqe->u0.dw[2] = 0; 2071 2072 #if defined(INET6) || defined(INET) 2073 if (IF_LRO_ENABLED(sc) && rq->lro_pkts_queued >= 16) { 2074 oce_rx_flush_lro(rq); 2075 } 2076 #endif 2077 2078 RING_GET(cq->ring, 1); 2079 bus_dmamap_sync(cq->ring->dma.tag, 2080 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2081 cqe = 2082 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); 2083 num_cqes++; 2084 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2085 break; 2086 } 2087 2088 #if defined(INET6) || defined(INET) 2089 if (IF_LRO_ENABLED(sc)) 2090 oce_rx_flush_lro(rq); 2091 #endif 2092 2093 oce_check_rx_bufs(sc, num_cqes, rq); 2094 UNLOCK(&rq->rx_lock); 2095 NET_EPOCH_EXIT(et); 2096 return 0; 2097 2098 } 2099 2100 /***************************************************************************** 2101 * Helper function prototypes in this file * 2102 *****************************************************************************/ 2103 2104 static int 2105 oce_attach_ifp(POCE_SOFTC sc) 2106 { 2107 2108 sc->ifp = if_alloc(IFT_ETHER); 2109 if (!sc->ifp) 2110 return ENOMEM; 2111 2112 ifmedia_init(&sc->media, IFM_IMASK, oce_media_change, oce_media_status); 2113 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2114 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 2115 2116 if_setflags(sc->ifp, IFF_BROADCAST | IFF_MULTICAST); 2117 if_setioctlfn(sc->ifp, oce_ioctl); 2118 if_setstartfn(sc->ifp, oce_start); 2119 if_setinitfn(sc->ifp, oce_init); 2120 if_setmtu(sc->ifp, ETHERMTU); 2121 if_setsoftc(sc->ifp, sc); 2122 if_settransmitfn(sc->ifp, oce_multiq_start); 2123 if_setqflushfn(sc->ifp, oce_multiq_flush); 2124 2125 if_initname(sc->ifp, 2126 device_get_name(sc->dev), device_get_unit(sc->dev)); 2127 2128 if_setsendqlen(sc->ifp, OCE_MAX_TX_DESC - 1); 2129 if_setsendqready(sc->ifp); 2130 2131 if_sethwassist(sc->ifp, OCE_IF_HWASSIST); 2132 if_sethwassistbits(sc->ifp, CSUM_TSO, 0); 2133 if_sethwassistbits(sc->ifp, (CSUM_IP | CSUM_TCP | CSUM_UDP), 0); 2134 2135 if_setcapabilities(sc->ifp, OCE_IF_CAPABILITIES); 2136 if_setcapabilitiesbit(sc->ifp, IFCAP_HWCSUM, 0); 2137 if_setcapabilitiesbit(sc->ifp, IFCAP_VLAN_HWFILTER, 0); 2138 2139 #if defined(INET6) || defined(INET) 2140 if_setcapabilitiesbit(sc->ifp, IFCAP_TSO, 0); 2141 if_setcapabilitiesbit(sc->ifp, IFCAP_LRO, 0); 2142 if_setcapabilitiesbit(sc->ifp, IFCAP_VLAN_HWTSO, 0); 2143 #endif 2144 2145 if_setcapenable(sc->ifp, if_getcapabilities(sc->ifp)); 2146 if_setbaudrate(sc->ifp, IF_Gbps(10)); 2147 2148 if_sethwtsomax(sc->ifp, 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN)); 2149 if_sethwtsomaxsegcount(sc->ifp, OCE_MAX_TX_ELEMENTS); 2150 if_sethwtsomaxsegsize(sc->ifp, 4096); 2151 2152 ether_ifattach(sc->ifp, sc->macaddr.mac_addr); 2153 2154 return 0; 2155 } 2156 2157 static void 2158 oce_add_vlan(void *arg, if_t ifp, uint16_t vtag) 2159 { 2160 POCE_SOFTC sc = if_getsoftc(ifp); 2161 2162 if (if_getsoftc(ifp) != arg) 2163 return; 2164 if ((vtag == 0) || (vtag > 4095)) 2165 return; 2166 2167 sc->vlan_tag[vtag] = 1; 2168 sc->vlans_added++; 2169 if (sc->vlans_added <= (sc->max_vlans + 1)) 2170 oce_vid_config(sc); 2171 } 2172 2173 static void 2174 oce_del_vlan(void *arg, if_t ifp, uint16_t vtag) 2175 { 2176 POCE_SOFTC sc = if_getsoftc(ifp); 2177 2178 if (if_getsoftc(ifp) != arg) 2179 return; 2180 if ((vtag == 0) || (vtag > 4095)) 2181 return; 2182 2183 sc->vlan_tag[vtag] = 0; 2184 sc->vlans_added--; 2185 oce_vid_config(sc); 2186 } 2187 2188 /* 2189 * A max of 64 vlans can be configured in BE. If the user configures 2190 * more, place the card in vlan promiscuous mode. 2191 */ 2192 static int 2193 oce_vid_config(POCE_SOFTC sc) 2194 { 2195 struct normal_vlan vtags[MAX_VLANFILTER_SIZE]; 2196 uint16_t ntags = 0, i; 2197 int status = 0; 2198 2199 if ((sc->vlans_added <= MAX_VLANFILTER_SIZE) && 2200 (if_getcapenable(sc->ifp) & IFCAP_VLAN_HWFILTER)) { 2201 for (i = 0; i < MAX_VLANS; i++) { 2202 if (sc->vlan_tag[i]) { 2203 vtags[ntags].vtag = i; 2204 ntags++; 2205 } 2206 } 2207 if (ntags) 2208 status = oce_config_vlan(sc, (uint8_t) sc->if_id, 2209 vtags, ntags, 1, 0); 2210 } else 2211 status = oce_config_vlan(sc, (uint8_t) sc->if_id, 2212 NULL, 0, 1, 1); 2213 return status; 2214 } 2215 2216 static void 2217 oce_mac_addr_set(POCE_SOFTC sc) 2218 { 2219 uint32_t old_pmac_id = sc->pmac_id; 2220 int status = 0; 2221 2222 status = bcmp((if_getlladdr(sc->ifp)), sc->macaddr.mac_addr, 2223 sc->macaddr.size_of_struct); 2224 if (!status) 2225 return; 2226 2227 status = oce_mbox_macaddr_add(sc, (uint8_t *)(if_getlladdr(sc->ifp)), 2228 sc->if_id, &sc->pmac_id); 2229 if (!status) { 2230 status = oce_mbox_macaddr_del(sc, sc->if_id, old_pmac_id); 2231 bcopy((if_getlladdr(sc->ifp)), sc->macaddr.mac_addr, 2232 sc->macaddr.size_of_struct); 2233 } 2234 if (status) 2235 device_printf(sc->dev, "Failed update macaddress\n"); 2236 2237 } 2238 2239 static int 2240 oce_handle_passthrough(if_t ifp, caddr_t data) 2241 { 2242 POCE_SOFTC sc = if_getsoftc(ifp); 2243 struct ifreq *ifr = (struct ifreq *)data; 2244 int rc = ENXIO; 2245 char cookie[32] = {0}; 2246 void *priv_data = ifr_data_get_ptr(ifr); 2247 void *ioctl_ptr; 2248 uint32_t req_size; 2249 struct mbx_hdr req; 2250 OCE_DMA_MEM dma_mem; 2251 2252 if (copyin(priv_data, cookie, strlen(IOCTL_COOKIE))) 2253 return EFAULT; 2254 2255 if (memcmp(cookie, IOCTL_COOKIE, strlen(IOCTL_COOKIE))) 2256 return EINVAL; 2257 2258 ioctl_ptr = (char *)priv_data + strlen(IOCTL_COOKIE); 2259 if (copyin(ioctl_ptr, &req, sizeof(struct mbx_hdr))) 2260 return EFAULT; 2261 2262 req_size = le32toh(req.u0.req.request_length); 2263 if (req_size > 65536) 2264 return EINVAL; 2265 2266 req_size += sizeof(struct mbx_hdr); 2267 rc = oce_dma_alloc(sc, req_size, &dma_mem, 0); 2268 if (rc) 2269 return ENOMEM; 2270 2271 if (copyin(ioctl_ptr, OCE_DMAPTR(&dma_mem,char), req_size)) { 2272 rc = EFAULT; 2273 goto dma_free; 2274 } 2275 2276 rc = oce_pass_through_mbox(sc, &dma_mem, req_size); 2277 if (rc) { 2278 rc = EIO; 2279 goto dma_free; 2280 } 2281 2282 if (copyout(OCE_DMAPTR(&dma_mem,char), ioctl_ptr, req_size)) { 2283 rc = EFAULT; 2284 goto dma_free; 2285 } 2286 2287 /* 2288 firmware is filling all the attributes for this ioctl except 2289 the driver version..so fill it 2290 */ 2291 if(req.u0.rsp.opcode == OPCODE_COMMON_GET_CNTL_ATTRIBUTES) { 2292 struct mbx_common_get_cntl_attr *fw_cmd = 2293 (struct mbx_common_get_cntl_attr *)ioctl_ptr; 2294 _Static_assert(sizeof(COMPONENT_REVISION) <= 2295 sizeof(fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str), 2296 "driver version string too long"); 2297 2298 rc = copyout(COMPONENT_REVISION, 2299 fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str, 2300 sizeof(COMPONENT_REVISION)); 2301 } 2302 2303 dma_free: 2304 oce_dma_free(sc, &dma_mem); 2305 return rc; 2306 2307 } 2308 2309 static void 2310 oce_eqd_set_periodic(POCE_SOFTC sc) 2311 { 2312 struct oce_set_eqd set_eqd[OCE_MAX_EQ]; 2313 struct oce_aic_obj *aic; 2314 struct oce_eq *eqo; 2315 uint64_t now = 0, delta; 2316 int eqd, i, num = 0; 2317 uint32_t tx_reqs = 0, rxpkts = 0, pps; 2318 struct oce_wq *wq; 2319 struct oce_rq *rq; 2320 2321 #define ticks_to_msecs(t) (1000 * (t) / hz) 2322 2323 for (i = 0 ; i < sc->neqs; i++) { 2324 eqo = sc->eq[i]; 2325 aic = &sc->aic_obj[i]; 2326 /* When setting the static eq delay from the user space */ 2327 if (!aic->enable) { 2328 if (aic->ticks) 2329 aic->ticks = 0; 2330 eqd = aic->et_eqd; 2331 goto modify_eqd; 2332 } 2333 2334 if (i == 0) { 2335 rq = sc->rq[0]; 2336 rxpkts = rq->rx_stats.rx_pkts; 2337 } else 2338 rxpkts = 0; 2339 if (i + 1 < sc->nrqs) { 2340 rq = sc->rq[i + 1]; 2341 rxpkts += rq->rx_stats.rx_pkts; 2342 } 2343 if (i < sc->nwqs) { 2344 wq = sc->wq[i]; 2345 tx_reqs = wq->tx_stats.tx_reqs; 2346 } else 2347 tx_reqs = 0; 2348 now = ticks; 2349 2350 if (!aic->ticks || now < aic->ticks || 2351 rxpkts < aic->prev_rxpkts || tx_reqs < aic->prev_txreqs) { 2352 aic->prev_rxpkts = rxpkts; 2353 aic->prev_txreqs = tx_reqs; 2354 aic->ticks = now; 2355 continue; 2356 } 2357 2358 delta = ticks_to_msecs(now - aic->ticks); 2359 2360 pps = (((uint32_t)(rxpkts - aic->prev_rxpkts) * 1000) / delta) + 2361 (((uint32_t)(tx_reqs - aic->prev_txreqs) * 1000) / delta); 2362 eqd = (pps / 15000) << 2; 2363 if (eqd < 8) 2364 eqd = 0; 2365 2366 /* Make sure that the eq delay is in the known range */ 2367 eqd = min(eqd, aic->max_eqd); 2368 eqd = max(eqd, aic->min_eqd); 2369 2370 aic->prev_rxpkts = rxpkts; 2371 aic->prev_txreqs = tx_reqs; 2372 aic->ticks = now; 2373 2374 modify_eqd: 2375 if (eqd != aic->cur_eqd) { 2376 set_eqd[num].delay_multiplier = (eqd * 65)/100; 2377 set_eqd[num].eq_id = eqo->eq_id; 2378 aic->cur_eqd = eqd; 2379 num++; 2380 } 2381 } 2382 2383 /* Is there atleast one eq that needs to be modified? */ 2384 for(i = 0; i < num; i += 8) { 2385 if((num - i) >=8 ) 2386 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], 8); 2387 else 2388 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], (num - i)); 2389 } 2390 2391 } 2392 2393 static void oce_detect_hw_error(POCE_SOFTC sc) 2394 { 2395 2396 uint32_t ue_low = 0, ue_high = 0, ue_low_mask = 0, ue_high_mask = 0; 2397 uint32_t sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0; 2398 uint32_t i; 2399 2400 if (sc->hw_error) 2401 return; 2402 2403 if (IS_XE201(sc)) { 2404 sliport_status = OCE_READ_REG32(sc, db, SLIPORT_STATUS_OFFSET); 2405 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2406 sliport_err1 = OCE_READ_REG32(sc, db, SLIPORT_ERROR1_OFFSET); 2407 sliport_err2 = OCE_READ_REG32(sc, db, SLIPORT_ERROR2_OFFSET); 2408 } 2409 } else { 2410 ue_low = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW); 2411 ue_high = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HIGH); 2412 ue_low_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW_MASK); 2413 ue_high_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HI_MASK); 2414 2415 ue_low = (ue_low & ~ue_low_mask); 2416 ue_high = (ue_high & ~ue_high_mask); 2417 } 2418 2419 /* On certain platforms BE hardware can indicate spurious UEs. 2420 * Allow the h/w to stop working completely in case of a real UE. 2421 * Hence not setting the hw_error for UE detection. 2422 */ 2423 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2424 sc->hw_error = TRUE; 2425 device_printf(sc->dev, "Error detected in the card\n"); 2426 } 2427 2428 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2429 device_printf(sc->dev, 2430 "ERR: sliport status 0x%x\n", sliport_status); 2431 device_printf(sc->dev, 2432 "ERR: sliport error1 0x%x\n", sliport_err1); 2433 device_printf(sc->dev, 2434 "ERR: sliport error2 0x%x\n", sliport_err2); 2435 } 2436 2437 if (ue_low) { 2438 for (i = 0; ue_low; ue_low >>= 1, i++) { 2439 if (ue_low & 1) 2440 device_printf(sc->dev, "UE: %s bit set\n", 2441 ue_status_low_desc[i]); 2442 } 2443 } 2444 2445 if (ue_high) { 2446 for (i = 0; ue_high; ue_high >>= 1, i++) { 2447 if (ue_high & 1) 2448 device_printf(sc->dev, "UE: %s bit set\n", 2449 ue_status_hi_desc[i]); 2450 } 2451 } 2452 2453 } 2454 2455 static void 2456 oce_local_timer(void *arg) 2457 { 2458 POCE_SOFTC sc = arg; 2459 int i = 0; 2460 2461 oce_detect_hw_error(sc); 2462 oce_refresh_nic_stats(sc); 2463 oce_refresh_queue_stats(sc); 2464 oce_mac_addr_set(sc); 2465 2466 /* TX Watch Dog*/ 2467 for (i = 0; i < sc->nwqs; i++) 2468 oce_tx_restart(sc, sc->wq[i]); 2469 2470 /* calculate and set the eq delay for optimal interrupt rate */ 2471 if (IS_BE(sc) || IS_SH(sc)) 2472 oce_eqd_set_periodic(sc); 2473 2474 callout_reset(&sc->timer, hz, oce_local_timer, sc); 2475 } 2476 2477 static void 2478 oce_tx_compl_clean(POCE_SOFTC sc) 2479 { 2480 struct oce_wq *wq; 2481 int i = 0, timeo = 0, num_wqes = 0; 2482 int pending_txqs = sc->nwqs; 2483 2484 /* Stop polling for compls when HW has been silent for 10ms or 2485 * hw_error or no outstanding completions expected 2486 */ 2487 do { 2488 pending_txqs = sc->nwqs; 2489 2490 for_all_wq_queues(sc, wq, i) { 2491 num_wqes = oce_wq_handler(wq); 2492 2493 if(num_wqes) 2494 timeo = 0; 2495 2496 if(!wq->ring->num_used) 2497 pending_txqs--; 2498 } 2499 2500 if (pending_txqs == 0 || ++timeo > 10 || sc->hw_error) 2501 break; 2502 2503 DELAY(1000); 2504 } while (TRUE); 2505 2506 for_all_wq_queues(sc, wq, i) { 2507 while(wq->ring->num_used) { 2508 LOCK(&wq->tx_compl_lock); 2509 oce_process_tx_completion(wq); 2510 UNLOCK(&wq->tx_compl_lock); 2511 } 2512 } 2513 2514 } 2515 2516 /* NOTE : This should only be called holding 2517 * DEVICE_LOCK. 2518 */ 2519 static void 2520 oce_if_deactivate(POCE_SOFTC sc) 2521 { 2522 int i; 2523 struct oce_rq *rq; 2524 struct oce_wq *wq; 2525 struct oce_eq *eq; 2526 2527 if_setdrvflagbits(sc->ifp, 0, (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)); 2528 2529 oce_tx_compl_clean(sc); 2530 2531 /* Stop intrs and finish any bottom halves pending */ 2532 oce_hw_intr_disable(sc); 2533 2534 /* Since taskqueue_drain takes a Gaint Lock, We should not acquire 2535 any other lock. So unlock device lock and require after 2536 completing taskqueue_drain. 2537 */ 2538 UNLOCK(&sc->dev_lock); 2539 for (i = 0; i < sc->intr_count; i++) { 2540 if (sc->intrs[i].tq != NULL) { 2541 taskqueue_drain(sc->intrs[i].tq, &sc->intrs[i].task); 2542 } 2543 } 2544 LOCK(&sc->dev_lock); 2545 2546 /* Delete RX queue in card with flush param */ 2547 oce_stop_rx(sc); 2548 2549 /* Invalidate any pending cq and eq entries*/ 2550 for_all_evnt_queues(sc, eq, i) 2551 oce_drain_eq(eq); 2552 for_all_rq_queues(sc, rq, i) 2553 oce_drain_rq_cq(rq); 2554 for_all_wq_queues(sc, wq, i) 2555 oce_drain_wq_cq(wq); 2556 2557 /* But still we need to get MCC aync events. 2558 So enable intrs and also arm first EQ 2559 */ 2560 oce_hw_intr_enable(sc); 2561 oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE); 2562 2563 DELAY(10); 2564 } 2565 2566 static void 2567 oce_if_activate(POCE_SOFTC sc) 2568 { 2569 struct oce_eq *eq; 2570 struct oce_rq *rq; 2571 struct oce_wq *wq; 2572 int i, rc = 0; 2573 2574 if_setdrvflagbits(sc->ifp, IFF_DRV_RUNNING , 0); 2575 2576 oce_hw_intr_disable(sc); 2577 2578 oce_start_rx(sc); 2579 2580 for_all_rq_queues(sc, rq, i) { 2581 rc = oce_start_rq(rq); 2582 if (rc) 2583 device_printf(sc->dev, "Unable to start RX\n"); 2584 } 2585 2586 for_all_wq_queues(sc, wq, i) { 2587 rc = oce_start_wq(wq); 2588 if (rc) 2589 device_printf(sc->dev, "Unable to start TX\n"); 2590 } 2591 2592 for_all_evnt_queues(sc, eq, i) 2593 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); 2594 2595 oce_hw_intr_enable(sc); 2596 2597 } 2598 2599 static void 2600 process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe) 2601 { 2602 /* Update Link status */ 2603 if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) == 2604 ASYNC_EVENT_LINK_UP) { 2605 sc->link_status = ASYNC_EVENT_LINK_UP; 2606 if_link_state_change(sc->ifp, LINK_STATE_UP); 2607 } else { 2608 sc->link_status = ASYNC_EVENT_LINK_DOWN; 2609 if_link_state_change(sc->ifp, LINK_STATE_DOWN); 2610 } 2611 } 2612 2613 static void oce_async_grp5_osbmc_process(POCE_SOFTC sc, 2614 struct oce_async_evt_grp5_os2bmc *evt) 2615 { 2616 DW_SWAP(evt, sizeof(struct oce_async_evt_grp5_os2bmc)); 2617 if (evt->u.s.mgmt_enable) 2618 sc->flags |= OCE_FLAGS_OS2BMC; 2619 else 2620 return; 2621 2622 sc->bmc_filt_mask = evt->u.s.arp_filter; 2623 sc->bmc_filt_mask |= (evt->u.s.dhcp_client_filt << 1); 2624 sc->bmc_filt_mask |= (evt->u.s.dhcp_server_filt << 2); 2625 sc->bmc_filt_mask |= (evt->u.s.net_bios_filt << 3); 2626 sc->bmc_filt_mask |= (evt->u.s.bcast_filt << 4); 2627 sc->bmc_filt_mask |= (evt->u.s.ipv6_nbr_filt << 5); 2628 sc->bmc_filt_mask |= (evt->u.s.ipv6_ra_filt << 6); 2629 sc->bmc_filt_mask |= (evt->u.s.ipv6_ras_filt << 7); 2630 sc->bmc_filt_mask |= (evt->u.s.mcast_filt << 8); 2631 } 2632 2633 static void oce_process_grp5_events(POCE_SOFTC sc, struct oce_mq_cqe *cqe) 2634 { 2635 struct oce_async_event_grp5_pvid_state *gcqe; 2636 struct oce_async_evt_grp5_os2bmc *bmccqe; 2637 2638 switch (cqe->u0.s.async_type) { 2639 case ASYNC_EVENT_PVID_STATE: 2640 /* GRP5 PVID */ 2641 gcqe = (struct oce_async_event_grp5_pvid_state *)cqe; 2642 if (gcqe->enabled) 2643 sc->pvid = gcqe->tag & VLAN_VID_MASK; 2644 else 2645 sc->pvid = 0; 2646 break; 2647 case ASYNC_EVENT_OS2BMC: 2648 bmccqe = (struct oce_async_evt_grp5_os2bmc *)cqe; 2649 oce_async_grp5_osbmc_process(sc, bmccqe); 2650 break; 2651 default: 2652 break; 2653 } 2654 } 2655 2656 /* Handle the Completion Queue for the Mailbox/Async notifications */ 2657 uint16_t 2658 oce_mq_handler(void *arg) 2659 { 2660 struct oce_mq *mq = (struct oce_mq *)arg; 2661 POCE_SOFTC sc = mq->parent; 2662 struct oce_cq *cq = mq->cq; 2663 int num_cqes = 0, evt_type = 0, optype = 0; 2664 struct oce_mq_cqe *cqe; 2665 struct oce_async_cqe_link_state *acqe; 2666 struct oce_async_event_qnq *dbgcqe; 2667 2668 bus_dmamap_sync(cq->ring->dma.tag, 2669 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2670 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe); 2671 2672 while (cqe->u0.dw[3]) { 2673 DW_SWAP((uint32_t *) cqe, sizeof(oce_mq_cqe)); 2674 if (cqe->u0.s.async_event) { 2675 evt_type = cqe->u0.s.event_type; 2676 optype = cqe->u0.s.async_type; 2677 if (evt_type == ASYNC_EVENT_CODE_LINK_STATE) { 2678 /* Link status evt */ 2679 acqe = (struct oce_async_cqe_link_state *)cqe; 2680 process_link_state(sc, acqe); 2681 } else if (evt_type == ASYNC_EVENT_GRP5) { 2682 oce_process_grp5_events(sc, cqe); 2683 } else if (evt_type == ASYNC_EVENT_CODE_DEBUG && 2684 optype == ASYNC_EVENT_DEBUG_QNQ) { 2685 dbgcqe = (struct oce_async_event_qnq *)cqe; 2686 if(dbgcqe->valid) 2687 sc->qnqid = dbgcqe->vlan_tag; 2688 sc->qnq_debug_event = TRUE; 2689 } 2690 } 2691 cqe->u0.dw[3] = 0; 2692 RING_GET(cq->ring, 1); 2693 bus_dmamap_sync(cq->ring->dma.tag, 2694 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2695 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe); 2696 num_cqes++; 2697 } 2698 2699 if (num_cqes) 2700 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); 2701 2702 return 0; 2703 } 2704 2705 static void 2706 setup_max_queues_want(POCE_SOFTC sc) 2707 { 2708 /* Check if it is FLEX machine. Is so dont use RSS */ 2709 if ((sc->function_mode & FNM_FLEX10_MODE) || 2710 (sc->function_mode & FNM_UMC_MODE) || 2711 (sc->function_mode & FNM_VNIC_MODE) || 2712 (!is_rss_enabled(sc)) || 2713 IS_BE2(sc)) { 2714 sc->nrqs = 1; 2715 sc->nwqs = 1; 2716 } else { 2717 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1; 2718 sc->nwqs = MIN(OCE_NCPUS, sc->nrssqs); 2719 } 2720 2721 if (IS_BE2(sc) && is_rss_enabled(sc)) 2722 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1; 2723 } 2724 2725 static void 2726 update_queues_got(POCE_SOFTC sc) 2727 { 2728 if (is_rss_enabled(sc)) { 2729 sc->nrqs = sc->intr_count + 1; 2730 sc->nwqs = sc->intr_count; 2731 } else { 2732 sc->nrqs = 1; 2733 sc->nwqs = 1; 2734 } 2735 2736 if (IS_BE2(sc)) 2737 sc->nwqs = 1; 2738 } 2739 2740 static int 2741 oce_check_ipv6_ext_hdr(struct mbuf *m) 2742 { 2743 struct ether_header *eh = mtod(m, struct ether_header *); 2744 caddr_t m_datatemp = m->m_data; 2745 2746 if (eh->ether_type == htons(ETHERTYPE_IPV6)) { 2747 m->m_data += sizeof(struct ether_header); 2748 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); 2749 2750 if((ip6->ip6_nxt != IPPROTO_TCP) && \ 2751 (ip6->ip6_nxt != IPPROTO_UDP)){ 2752 struct ip6_ext *ip6e = NULL; 2753 m->m_data += sizeof(struct ip6_hdr); 2754 2755 ip6e = (struct ip6_ext *) mtod(m, struct ip6_ext *); 2756 if(ip6e->ip6e_len == 0xff) { 2757 m->m_data = m_datatemp; 2758 return TRUE; 2759 } 2760 } 2761 m->m_data = m_datatemp; 2762 } 2763 return FALSE; 2764 } 2765 2766 static int 2767 is_be3_a1(POCE_SOFTC sc) 2768 { 2769 if((sc->flags & OCE_FLAGS_BE3) && ((sc->asic_revision & 0xFF) < 2)) { 2770 return TRUE; 2771 } 2772 return FALSE; 2773 } 2774 2775 static struct mbuf * 2776 oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete) 2777 { 2778 uint16_t vlan_tag = 0; 2779 2780 if(!M_WRITABLE(m)) 2781 return NULL; 2782 2783 /* Embed vlan tag in the packet if it is not part of it */ 2784 if(m->m_flags & M_VLANTAG) { 2785 vlan_tag = EVL_VLANOFTAG(m->m_pkthdr.ether_vtag); 2786 m->m_flags &= ~M_VLANTAG; 2787 } 2788 2789 /* if UMC, ignore vlan tag insertion and instead insert pvid */ 2790 if(sc->pvid) { 2791 if(!vlan_tag) 2792 vlan_tag = sc->pvid; 2793 if (complete) 2794 *complete = FALSE; 2795 } 2796 2797 if(vlan_tag) { 2798 m = ether_vlanencap(m, vlan_tag); 2799 } 2800 2801 if(sc->qnqid) { 2802 m = ether_vlanencap(m, sc->qnqid); 2803 2804 if (complete) 2805 *complete = FALSE; 2806 } 2807 return m; 2808 } 2809 2810 static int 2811 oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m) 2812 { 2813 if(is_be3_a1(sc) && IS_QNQ_OR_UMC(sc) && \ 2814 oce_check_ipv6_ext_hdr(m)) { 2815 return TRUE; 2816 } 2817 return FALSE; 2818 } 2819 2820 static void 2821 oce_get_config(POCE_SOFTC sc) 2822 { 2823 int rc = 0; 2824 uint32_t max_rss = 0; 2825 2826 if ((IS_BE(sc) || IS_SH(sc)) && (!sc->be3_native)) 2827 max_rss = OCE_LEGACY_MODE_RSS; 2828 else 2829 max_rss = OCE_MAX_RSS; 2830 2831 if (!IS_BE(sc)) { 2832 rc = oce_get_profile_config(sc, max_rss); 2833 if (rc) { 2834 sc->nwqs = OCE_MAX_WQ; 2835 sc->nrssqs = max_rss; 2836 sc->nrqs = sc->nrssqs + 1; 2837 } 2838 } 2839 else { /* For BE3 don't rely on fw for determining the resources */ 2840 sc->nrssqs = max_rss; 2841 sc->nrqs = sc->nrssqs + 1; 2842 sc->nwqs = OCE_MAX_WQ; 2843 sc->max_vlans = MAX_VLANFILTER_SIZE; 2844 } 2845 } 2846 2847 static void 2848 oce_rdma_close(void) 2849 { 2850 if (oce_rdma_if != NULL) { 2851 oce_rdma_if = NULL; 2852 } 2853 } 2854 2855 static void 2856 oce_get_mac_addr(POCE_SOFTC sc, uint8_t *macaddr) 2857 { 2858 memcpy(macaddr, sc->macaddr.mac_addr, 6); 2859 } 2860 2861 int 2862 oce_register_rdma(POCE_RDMA_INFO rdma_info, POCE_RDMA_IF rdma_if) 2863 { 2864 POCE_SOFTC sc; 2865 struct oce_dev_info di; 2866 int i; 2867 2868 if ((rdma_info == NULL) || (rdma_if == NULL)) { 2869 return -EINVAL; 2870 } 2871 2872 if ((rdma_info->size != OCE_RDMA_INFO_SIZE) || 2873 (rdma_if->size != OCE_RDMA_IF_SIZE)) { 2874 return -ENXIO; 2875 } 2876 2877 rdma_info->close = oce_rdma_close; 2878 rdma_info->mbox_post = oce_mbox_post; 2879 rdma_info->common_req_hdr_init = mbx_common_req_hdr_init; 2880 rdma_info->get_mac_addr = oce_get_mac_addr; 2881 2882 oce_rdma_if = rdma_if; 2883 2884 sc = softc_head; 2885 while (sc != NULL) { 2886 if (oce_rdma_if->announce != NULL) { 2887 memset(&di, 0, sizeof(di)); 2888 di.dev = sc->dev; 2889 di.softc = sc; 2890 di.ifp = sc->ifp; 2891 di.db_bhandle = sc->db_bhandle; 2892 di.db_btag = sc->db_btag; 2893 di.db_page_size = 4096; 2894 if (sc->flags & OCE_FLAGS_USING_MSIX) { 2895 di.intr_mode = OCE_INTERRUPT_MODE_MSIX; 2896 } else if (sc->flags & OCE_FLAGS_USING_MSI) { 2897 di.intr_mode = OCE_INTERRUPT_MODE_MSI; 2898 } else { 2899 di.intr_mode = OCE_INTERRUPT_MODE_INTX; 2900 } 2901 di.dev_family = OCE_GEN2_FAMILY; // fixme: must detect skyhawk 2902 if (di.intr_mode != OCE_INTERRUPT_MODE_INTX) { 2903 di.msix.num_vectors = sc->intr_count + sc->roce_intr_count; 2904 di.msix.start_vector = sc->intr_count; 2905 for (i=0; i<di.msix.num_vectors; i++) { 2906 di.msix.vector_list[i] = sc->intrs[i].vector; 2907 } 2908 } else { 2909 } 2910 memcpy(di.mac_addr, sc->macaddr.mac_addr, 6); 2911 di.vendor_id = pci_get_vendor(sc->dev); 2912 di.dev_id = pci_get_device(sc->dev); 2913 2914 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 2915 di.flags |= OCE_RDMA_INFO_RDMA_SUPPORTED; 2916 } 2917 2918 rdma_if->announce(&di); 2919 sc = sc->next; 2920 } 2921 } 2922 2923 return 0; 2924 } 2925 2926 static void 2927 oce_read_env_variables( POCE_SOFTC sc ) 2928 { 2929 char *value = NULL; 2930 int rc = 0; 2931 2932 /* read if user wants to enable hwlro or swlro */ 2933 //value = getenv("oce_enable_hwlro"); 2934 if(value && IS_SH(sc)) { 2935 sc->enable_hwlro = strtol(value, NULL, 10); 2936 if(sc->enable_hwlro) { 2937 rc = oce_mbox_nic_query_lro_capabilities(sc, NULL, NULL); 2938 if(rc) { 2939 device_printf(sc->dev, "no hardware lro support\n"); 2940 device_printf(sc->dev, "software lro enabled\n"); 2941 sc->enable_hwlro = 0; 2942 }else { 2943 device_printf(sc->dev, "hardware lro enabled\n"); 2944 oce_max_rsp_handled = 32; 2945 } 2946 }else { 2947 device_printf(sc->dev, "software lro enabled\n"); 2948 } 2949 }else { 2950 sc->enable_hwlro = 0; 2951 } 2952 2953 /* read mbuf size */ 2954 //value = getenv("oce_rq_buf_size"); 2955 if(value && IS_SH(sc)) { 2956 oce_rq_buf_size = strtol(value, NULL, 10); 2957 switch(oce_rq_buf_size) { 2958 case 2048: 2959 case 4096: 2960 case 9216: 2961 case 16384: 2962 break; 2963 2964 default: 2965 device_printf(sc->dev, " Supported oce_rq_buf_size values are 2K, 4K, 9K, 16K \n"); 2966 oce_rq_buf_size = 2048; 2967 } 2968 } 2969 2970 return; 2971 } 2972