1 /****************************************************************************** 2 3 Copyright (c) 2006-2013, Myricom Inc. 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Neither the name of the Myricom Inc, nor the names of its 13 contributors may be used to endorse or promote products derived from 14 this software without specific prior written permission. 15 16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 POSSIBILITY OF SUCH DAMAGE. 27 28 ***************************************************************************/ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/linker.h> 36 #include <sys/firmware.h> 37 #include <sys/endian.h> 38 #include <sys/sockio.h> 39 #include <sys/mbuf.h> 40 #include <sys/malloc.h> 41 #include <sys/kdb.h> 42 #include <sys/kernel.h> 43 #include <sys/lock.h> 44 #include <sys/module.h> 45 #include <sys/socket.h> 46 #include <sys/sysctl.h> 47 #include <sys/sx.h> 48 #include <sys/taskqueue.h> 49 50 #include <net/if.h> 51 #include <net/if_arp.h> 52 #include <net/ethernet.h> 53 #include <net/if_dl.h> 54 #include <net/if_media.h> 55 56 #include <net/bpf.h> 57 58 #include <net/if_types.h> 59 #include <net/if_vlan_var.h> 60 #include <net/zlib.h> 61 62 #include <netinet/in_systm.h> 63 #include <netinet/in.h> 64 #include <netinet/ip.h> 65 #include <netinet/ip6.h> 66 #include <netinet/tcp.h> 67 #include <netinet/tcp_lro.h> 68 #include <netinet6/ip6_var.h> 69 70 #include <machine/bus.h> 71 #include <machine/in_cksum.h> 72 #include <machine/resource.h> 73 #include <sys/bus.h> 74 #include <sys/rman.h> 75 #include <sys/smp.h> 76 77 #include <dev/pci/pcireg.h> 78 #include <dev/pci/pcivar.h> 79 #include <dev/pci/pci_private.h> /* XXX for pci_cfg_restore */ 80 81 #include <vm/vm.h> /* for pmap_mapdev() */ 82 #include <vm/pmap.h> 83 84 #if defined(__i386) || defined(__amd64) 85 #include <machine/specialreg.h> 86 #endif 87 88 #include <dev/mxge/mxge_mcp.h> 89 #include <dev/mxge/mcp_gen_header.h> 90 /*#define MXGE_FAKE_IFP*/ 91 #include <dev/mxge/if_mxge_var.h> 92 #ifdef IFNET_BUF_RING 93 #include <sys/buf_ring.h> 94 #endif 95 96 #include "opt_inet.h" 97 #include "opt_inet6.h" 98 99 /* tunable params */ 100 static int mxge_nvidia_ecrc_enable = 1; 101 static int mxge_force_firmware = 0; 102 static int mxge_intr_coal_delay = 30; 103 static int mxge_deassert_wait = 1; 104 static int mxge_flow_control = 1; 105 static int mxge_verbose = 0; 106 static int mxge_ticks; 107 static int mxge_max_slices = 1; 108 static int mxge_rss_hash_type = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT; 109 static int mxge_always_promisc = 0; 110 static int mxge_initial_mtu = ETHERMTU_JUMBO; 111 static int mxge_throttle = 0; 112 static char *mxge_fw_unaligned = "mxge_ethp_z8e"; 113 static char *mxge_fw_aligned = "mxge_eth_z8e"; 114 static char *mxge_fw_rss_aligned = "mxge_rss_eth_z8e"; 115 static char *mxge_fw_rss_unaligned = "mxge_rss_ethp_z8e"; 116 117 static int mxge_probe(device_t dev); 118 static int mxge_attach(device_t dev); 119 static int mxge_detach(device_t dev); 120 static int mxge_shutdown(device_t dev); 121 static void mxge_intr(void *arg); 122 123 static device_method_t mxge_methods[] = 124 { 125 /* Device interface */ 126 DEVMETHOD(device_probe, mxge_probe), 127 DEVMETHOD(device_attach, mxge_attach), 128 DEVMETHOD(device_detach, mxge_detach), 129 DEVMETHOD(device_shutdown, mxge_shutdown), 130 131 DEVMETHOD_END 132 }; 133 134 static driver_t mxge_driver = 135 { 136 "mxge", 137 mxge_methods, 138 sizeof(mxge_softc_t), 139 }; 140 141 static devclass_t mxge_devclass; 142 143 /* Declare ourselves to be a child of the PCI bus.*/ 144 DRIVER_MODULE(mxge, pci, mxge_driver, mxge_devclass, 0, 0); 145 MODULE_DEPEND(mxge, firmware, 1, 1, 1); 146 MODULE_DEPEND(mxge, zlib, 1, 1, 1); 147 148 static int mxge_load_firmware(mxge_softc_t *sc, int adopt); 149 static int mxge_send_cmd(mxge_softc_t *sc, uint32_t cmd, mxge_cmd_t *data); 150 static int mxge_close(mxge_softc_t *sc, int down); 151 static int mxge_open(mxge_softc_t *sc); 152 static void mxge_tick(void *arg); 153 154 static int 155 mxge_probe(device_t dev) 156 { 157 int rev; 158 159 160 if ((pci_get_vendor(dev) == MXGE_PCI_VENDOR_MYRICOM) && 161 ((pci_get_device(dev) == MXGE_PCI_DEVICE_Z8E) || 162 (pci_get_device(dev) == MXGE_PCI_DEVICE_Z8E_9))) { 163 rev = pci_get_revid(dev); 164 switch (rev) { 165 case MXGE_PCI_REV_Z8E: 166 device_set_desc(dev, "Myri10G-PCIE-8A"); 167 break; 168 case MXGE_PCI_REV_Z8ES: 169 device_set_desc(dev, "Myri10G-PCIE-8B"); 170 break; 171 default: 172 device_set_desc(dev, "Myri10G-PCIE-8??"); 173 device_printf(dev, "Unrecognized rev %d NIC\n", 174 rev); 175 break; 176 } 177 return 0; 178 } 179 return ENXIO; 180 } 181 182 static void 183 mxge_enable_wc(mxge_softc_t *sc) 184 { 185 #if defined(__i386) || defined(__amd64) 186 vm_offset_t len; 187 int err; 188 189 sc->wc = 1; 190 len = rman_get_size(sc->mem_res); 191 err = pmap_change_attr((vm_offset_t) sc->sram, 192 len, PAT_WRITE_COMBINING); 193 if (err != 0) { 194 device_printf(sc->dev, "pmap_change_attr failed, %d\n", 195 err); 196 sc->wc = 0; 197 } 198 #endif 199 } 200 201 202 /* callback to get our DMA address */ 203 static void 204 mxge_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nsegs, 205 int error) 206 { 207 if (error == 0) { 208 *(bus_addr_t *) arg = segs->ds_addr; 209 } 210 } 211 212 static int 213 mxge_dma_alloc(mxge_softc_t *sc, mxge_dma_t *dma, size_t bytes, 214 bus_size_t alignment) 215 { 216 int err; 217 device_t dev = sc->dev; 218 bus_size_t boundary, maxsegsize; 219 220 if (bytes > 4096 && alignment == 4096) { 221 boundary = 0; 222 maxsegsize = bytes; 223 } else { 224 boundary = 4096; 225 maxsegsize = 4096; 226 } 227 228 /* allocate DMAable memory tags */ 229 err = bus_dma_tag_create(sc->parent_dmat, /* parent */ 230 alignment, /* alignment */ 231 boundary, /* boundary */ 232 BUS_SPACE_MAXADDR, /* low */ 233 BUS_SPACE_MAXADDR, /* high */ 234 NULL, NULL, /* filter */ 235 bytes, /* maxsize */ 236 1, /* num segs */ 237 maxsegsize, /* maxsegsize */ 238 BUS_DMA_COHERENT, /* flags */ 239 NULL, NULL, /* lock */ 240 &dma->dmat); /* tag */ 241 if (err != 0) { 242 device_printf(dev, "couldn't alloc tag (err = %d)\n", err); 243 return err; 244 } 245 246 /* allocate DMAable memory & map */ 247 err = bus_dmamem_alloc(dma->dmat, &dma->addr, 248 (BUS_DMA_WAITOK | BUS_DMA_COHERENT 249 | BUS_DMA_ZERO), &dma->map); 250 if (err != 0) { 251 device_printf(dev, "couldn't alloc mem (err = %d)\n", err); 252 goto abort_with_dmat; 253 } 254 255 /* load the memory */ 256 err = bus_dmamap_load(dma->dmat, dma->map, dma->addr, bytes, 257 mxge_dmamap_callback, 258 (void *)&dma->bus_addr, 0); 259 if (err != 0) { 260 device_printf(dev, "couldn't load map (err = %d)\n", err); 261 goto abort_with_mem; 262 } 263 return 0; 264 265 abort_with_mem: 266 bus_dmamem_free(dma->dmat, dma->addr, dma->map); 267 abort_with_dmat: 268 (void)bus_dma_tag_destroy(dma->dmat); 269 return err; 270 } 271 272 273 static void 274 mxge_dma_free(mxge_dma_t *dma) 275 { 276 bus_dmamap_unload(dma->dmat, dma->map); 277 bus_dmamem_free(dma->dmat, dma->addr, dma->map); 278 (void)bus_dma_tag_destroy(dma->dmat); 279 } 280 281 /* 282 * The eeprom strings on the lanaiX have the format 283 * SN=x\0 284 * MAC=x:x:x:x:x:x\0 285 * PC=text\0 286 */ 287 288 static int 289 mxge_parse_strings(mxge_softc_t *sc) 290 { 291 char *ptr; 292 int i, found_mac, found_sn2; 293 char *endptr; 294 295 ptr = sc->eeprom_strings; 296 found_mac = 0; 297 found_sn2 = 0; 298 while (*ptr != '\0') { 299 if (strncmp(ptr, "MAC=", 4) == 0) { 300 ptr += 4; 301 for (i = 0;;) { 302 sc->mac_addr[i] = strtoul(ptr, &endptr, 16); 303 if (endptr - ptr != 2) 304 goto abort; 305 ptr = endptr; 306 if (++i == 6) 307 break; 308 if (*ptr++ != ':') 309 goto abort; 310 } 311 found_mac = 1; 312 } else if (strncmp(ptr, "PC=", 3) == 0) { 313 ptr += 3; 314 strlcpy(sc->product_code_string, ptr, 315 sizeof(sc->product_code_string)); 316 } else if (!found_sn2 && (strncmp(ptr, "SN=", 3) == 0)) { 317 ptr += 3; 318 strlcpy(sc->serial_number_string, ptr, 319 sizeof(sc->serial_number_string)); 320 } else if (strncmp(ptr, "SN2=", 4) == 0) { 321 /* SN2 takes precedence over SN */ 322 ptr += 4; 323 found_sn2 = 1; 324 strlcpy(sc->serial_number_string, ptr, 325 sizeof(sc->serial_number_string)); 326 } 327 while (*ptr++ != '\0') {} 328 } 329 330 if (found_mac) 331 return 0; 332 333 abort: 334 device_printf(sc->dev, "failed to parse eeprom_strings\n"); 335 336 return ENXIO; 337 } 338 339 #if defined __i386 || defined i386 || defined __i386__ || defined __x86_64__ 340 static void 341 mxge_enable_nvidia_ecrc(mxge_softc_t *sc) 342 { 343 uint32_t val; 344 unsigned long base, off; 345 char *va, *cfgptr; 346 device_t pdev, mcp55; 347 uint16_t vendor_id, device_id, word; 348 uintptr_t bus, slot, func, ivend, idev; 349 uint32_t *ptr32; 350 351 352 if (!mxge_nvidia_ecrc_enable) 353 return; 354 355 pdev = device_get_parent(device_get_parent(sc->dev)); 356 if (pdev == NULL) { 357 device_printf(sc->dev, "could not find parent?\n"); 358 return; 359 } 360 vendor_id = pci_read_config(pdev, PCIR_VENDOR, 2); 361 device_id = pci_read_config(pdev, PCIR_DEVICE, 2); 362 363 if (vendor_id != 0x10de) 364 return; 365 366 base = 0; 367 368 if (device_id == 0x005d) { 369 /* ck804, base address is magic */ 370 base = 0xe0000000UL; 371 } else if (device_id >= 0x0374 && device_id <= 0x378) { 372 /* mcp55, base address stored in chipset */ 373 mcp55 = pci_find_bsf(0, 0, 0); 374 if (mcp55 && 375 0x10de == pci_read_config(mcp55, PCIR_VENDOR, 2) && 376 0x0369 == pci_read_config(mcp55, PCIR_DEVICE, 2)) { 377 word = pci_read_config(mcp55, 0x90, 2); 378 base = ((unsigned long)word & 0x7ffeU) << 25; 379 } 380 } 381 if (!base) 382 return; 383 384 /* XXXX 385 Test below is commented because it is believed that doing 386 config read/write beyond 0xff will access the config space 387 for the next larger function. Uncomment this and remove 388 the hacky pmap_mapdev() way of accessing config space when 389 FreeBSD grows support for extended pcie config space access 390 */ 391 #if 0 392 /* See if we can, by some miracle, access the extended 393 config space */ 394 val = pci_read_config(pdev, 0x178, 4); 395 if (val != 0xffffffff) { 396 val |= 0x40; 397 pci_write_config(pdev, 0x178, val, 4); 398 return; 399 } 400 #endif 401 /* Rather than using normal pci config space writes, we must 402 * map the Nvidia config space ourselves. This is because on 403 * opteron/nvidia class machine the 0xe000000 mapping is 404 * handled by the nvidia chipset, that means the internal PCI 405 * device (the on-chip northbridge), or the amd-8131 bridge 406 * and things behind them are not visible by this method. 407 */ 408 409 BUS_READ_IVAR(device_get_parent(pdev), pdev, 410 PCI_IVAR_BUS, &bus); 411 BUS_READ_IVAR(device_get_parent(pdev), pdev, 412 PCI_IVAR_SLOT, &slot); 413 BUS_READ_IVAR(device_get_parent(pdev), pdev, 414 PCI_IVAR_FUNCTION, &func); 415 BUS_READ_IVAR(device_get_parent(pdev), pdev, 416 PCI_IVAR_VENDOR, &ivend); 417 BUS_READ_IVAR(device_get_parent(pdev), pdev, 418 PCI_IVAR_DEVICE, &idev); 419 420 off = base 421 + 0x00100000UL * (unsigned long)bus 422 + 0x00001000UL * (unsigned long)(func 423 + 8 * slot); 424 425 /* map it into the kernel */ 426 va = pmap_mapdev(trunc_page((vm_paddr_t)off), PAGE_SIZE); 427 428 429 if (va == NULL) { 430 device_printf(sc->dev, "pmap_kenter_temporary didn't\n"); 431 return; 432 } 433 /* get a pointer to the config space mapped into the kernel */ 434 cfgptr = va + (off & PAGE_MASK); 435 436 /* make sure that we can really access it */ 437 vendor_id = *(uint16_t *)(cfgptr + PCIR_VENDOR); 438 device_id = *(uint16_t *)(cfgptr + PCIR_DEVICE); 439 if (! (vendor_id == ivend && device_id == idev)) { 440 device_printf(sc->dev, "mapping failed: 0x%x:0x%x\n", 441 vendor_id, device_id); 442 pmap_unmapdev((vm_offset_t)va, PAGE_SIZE); 443 return; 444 } 445 446 ptr32 = (uint32_t*)(cfgptr + 0x178); 447 val = *ptr32; 448 449 if (val == 0xffffffff) { 450 device_printf(sc->dev, "extended mapping failed\n"); 451 pmap_unmapdev((vm_offset_t)va, PAGE_SIZE); 452 return; 453 } 454 *ptr32 = val | 0x40; 455 pmap_unmapdev((vm_offset_t)va, PAGE_SIZE); 456 if (mxge_verbose) 457 device_printf(sc->dev, 458 "Enabled ECRC on upstream Nvidia bridge " 459 "at %d:%d:%d\n", 460 (int)bus, (int)slot, (int)func); 461 return; 462 } 463 #else 464 static void 465 mxge_enable_nvidia_ecrc(mxge_softc_t *sc) 466 { 467 device_printf(sc->dev, 468 "Nforce 4 chipset on non-x86/amd64!?!?!\n"); 469 return; 470 } 471 #endif 472 473 474 static int 475 mxge_dma_test(mxge_softc_t *sc, int test_type) 476 { 477 mxge_cmd_t cmd; 478 bus_addr_t dmatest_bus = sc->dmabench_dma.bus_addr; 479 int status; 480 uint32_t len; 481 char *test = " "; 482 483 484 /* Run a small DMA test. 485 * The magic multipliers to the length tell the firmware 486 * to do DMA read, write, or read+write tests. The 487 * results are returned in cmd.data0. The upper 16 488 * bits of the return is the number of transfers completed. 489 * The lower 16 bits is the time in 0.5us ticks that the 490 * transfers took to complete. 491 */ 492 493 len = sc->tx_boundary; 494 495 cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus); 496 cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus); 497 cmd.data2 = len * 0x10000; 498 status = mxge_send_cmd(sc, test_type, &cmd); 499 if (status != 0) { 500 test = "read"; 501 goto abort; 502 } 503 sc->read_dma = ((cmd.data0>>16) * len * 2) / 504 (cmd.data0 & 0xffff); 505 cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus); 506 cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus); 507 cmd.data2 = len * 0x1; 508 status = mxge_send_cmd(sc, test_type, &cmd); 509 if (status != 0) { 510 test = "write"; 511 goto abort; 512 } 513 sc->write_dma = ((cmd.data0>>16) * len * 2) / 514 (cmd.data0 & 0xffff); 515 516 cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus); 517 cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus); 518 cmd.data2 = len * 0x10001; 519 status = mxge_send_cmd(sc, test_type, &cmd); 520 if (status != 0) { 521 test = "read/write"; 522 goto abort; 523 } 524 sc->read_write_dma = ((cmd.data0>>16) * len * 2 * 2) / 525 (cmd.data0 & 0xffff); 526 527 abort: 528 if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST) 529 device_printf(sc->dev, "DMA %s benchmark failed: %d\n", 530 test, status); 531 532 return status; 533 } 534 535 /* 536 * The Lanai Z8E PCI-E interface achieves higher Read-DMA throughput 537 * when the PCI-E Completion packets are aligned on an 8-byte 538 * boundary. Some PCI-E chip sets always align Completion packets; on 539 * the ones that do not, the alignment can be enforced by enabling 540 * ECRC generation (if supported). 541 * 542 * When PCI-E Completion packets are not aligned, it is actually more 543 * efficient to limit Read-DMA transactions to 2KB, rather than 4KB. 544 * 545 * If the driver can neither enable ECRC nor verify that it has 546 * already been enabled, then it must use a firmware image which works 547 * around unaligned completion packets (ethp_z8e.dat), and it should 548 * also ensure that it never gives the device a Read-DMA which is 549 * larger than 2KB by setting the tx_boundary to 2KB. If ECRC is 550 * enabled, then the driver should use the aligned (eth_z8e.dat) 551 * firmware image, and set tx_boundary to 4KB. 552 */ 553 554 static int 555 mxge_firmware_probe(mxge_softc_t *sc) 556 { 557 device_t dev = sc->dev; 558 int reg, status; 559 uint16_t pectl; 560 561 sc->tx_boundary = 4096; 562 /* 563 * Verify the max read request size was set to 4KB 564 * before trying the test with 4KB. 565 */ 566 if (pci_find_cap(dev, PCIY_EXPRESS, ®) == 0) { 567 pectl = pci_read_config(dev, reg + 0x8, 2); 568 if ((pectl & (5 << 12)) != (5 << 12)) { 569 device_printf(dev, "Max Read Req. size != 4k (0x%x\n", 570 pectl); 571 sc->tx_boundary = 2048; 572 } 573 } 574 575 /* 576 * load the optimized firmware (which assumes aligned PCIe 577 * completions) in order to see if it works on this host. 578 */ 579 sc->fw_name = mxge_fw_aligned; 580 status = mxge_load_firmware(sc, 1); 581 if (status != 0) { 582 return status; 583 } 584 585 /* 586 * Enable ECRC if possible 587 */ 588 mxge_enable_nvidia_ecrc(sc); 589 590 /* 591 * Run a DMA test which watches for unaligned completions and 592 * aborts on the first one seen. Not required on Z8ES or newer. 593 */ 594 if (pci_get_revid(sc->dev) >= MXGE_PCI_REV_Z8ES) 595 return 0; 596 status = mxge_dma_test(sc, MXGEFW_CMD_UNALIGNED_TEST); 597 if (status == 0) 598 return 0; /* keep the aligned firmware */ 599 600 if (status != E2BIG) 601 device_printf(dev, "DMA test failed: %d\n", status); 602 if (status == ENOSYS) 603 device_printf(dev, "Falling back to ethp! " 604 "Please install up to date fw\n"); 605 return status; 606 } 607 608 static int 609 mxge_select_firmware(mxge_softc_t *sc) 610 { 611 int aligned = 0; 612 int force_firmware = mxge_force_firmware; 613 614 if (sc->throttle) 615 force_firmware = sc->throttle; 616 617 if (force_firmware != 0) { 618 if (force_firmware == 1) 619 aligned = 1; 620 else 621 aligned = 0; 622 if (mxge_verbose) 623 device_printf(sc->dev, 624 "Assuming %s completions (forced)\n", 625 aligned ? "aligned" : "unaligned"); 626 goto abort; 627 } 628 629 /* if the PCIe link width is 4 or less, we can use the aligned 630 firmware and skip any checks */ 631 if (sc->link_width != 0 && sc->link_width <= 4) { 632 device_printf(sc->dev, 633 "PCIe x%d Link, expect reduced performance\n", 634 sc->link_width); 635 aligned = 1; 636 goto abort; 637 } 638 639 if (0 == mxge_firmware_probe(sc)) 640 return 0; 641 642 abort: 643 if (aligned) { 644 sc->fw_name = mxge_fw_aligned; 645 sc->tx_boundary = 4096; 646 } else { 647 sc->fw_name = mxge_fw_unaligned; 648 sc->tx_boundary = 2048; 649 } 650 return (mxge_load_firmware(sc, 0)); 651 } 652 653 static int 654 mxge_validate_firmware(mxge_softc_t *sc, const mcp_gen_header_t *hdr) 655 { 656 657 658 if (be32toh(hdr->mcp_type) != MCP_TYPE_ETH) { 659 device_printf(sc->dev, "Bad firmware type: 0x%x\n", 660 be32toh(hdr->mcp_type)); 661 return EIO; 662 } 663 664 /* save firmware version for sysctl */ 665 strlcpy(sc->fw_version, hdr->version, sizeof(sc->fw_version)); 666 if (mxge_verbose) 667 device_printf(sc->dev, "firmware id: %s\n", hdr->version); 668 669 sscanf(sc->fw_version, "%d.%d.%d", &sc->fw_ver_major, 670 &sc->fw_ver_minor, &sc->fw_ver_tiny); 671 672 if (!(sc->fw_ver_major == MXGEFW_VERSION_MAJOR 673 && sc->fw_ver_minor == MXGEFW_VERSION_MINOR)) { 674 device_printf(sc->dev, "Found firmware version %s\n", 675 sc->fw_version); 676 device_printf(sc->dev, "Driver needs %d.%d\n", 677 MXGEFW_VERSION_MAJOR, MXGEFW_VERSION_MINOR); 678 return EINVAL; 679 } 680 return 0; 681 682 } 683 684 static void * 685 z_alloc(void *nil, u_int items, u_int size) 686 { 687 void *ptr; 688 689 ptr = malloc(items * size, M_TEMP, M_NOWAIT); 690 return ptr; 691 } 692 693 static void 694 z_free(void *nil, void *ptr) 695 { 696 free(ptr, M_TEMP); 697 } 698 699 700 static int 701 mxge_load_firmware_helper(mxge_softc_t *sc, uint32_t *limit) 702 { 703 z_stream zs; 704 char *inflate_buffer; 705 const struct firmware *fw; 706 const mcp_gen_header_t *hdr; 707 unsigned hdr_offset; 708 int status; 709 unsigned int i; 710 char dummy; 711 size_t fw_len; 712 713 fw = firmware_get(sc->fw_name); 714 if (fw == NULL) { 715 device_printf(sc->dev, "Could not find firmware image %s\n", 716 sc->fw_name); 717 return ENOENT; 718 } 719 720 721 722 /* setup zlib and decompress f/w */ 723 bzero(&zs, sizeof (zs)); 724 zs.zalloc = z_alloc; 725 zs.zfree = z_free; 726 status = inflateInit(&zs); 727 if (status != Z_OK) { 728 status = EIO; 729 goto abort_with_fw; 730 } 731 732 /* the uncompressed size is stored as the firmware version, 733 which would otherwise go unused */ 734 fw_len = (size_t) fw->version; 735 inflate_buffer = malloc(fw_len, M_TEMP, M_NOWAIT); 736 if (inflate_buffer == NULL) 737 goto abort_with_zs; 738 zs.avail_in = fw->datasize; 739 zs.next_in = __DECONST(char *, fw->data); 740 zs.avail_out = fw_len; 741 zs.next_out = inflate_buffer; 742 status = inflate(&zs, Z_FINISH); 743 if (status != Z_STREAM_END) { 744 device_printf(sc->dev, "zlib %d\n", status); 745 status = EIO; 746 goto abort_with_buffer; 747 } 748 749 /* check id */ 750 hdr_offset = htobe32(*(const uint32_t *) 751 (inflate_buffer + MCP_HEADER_PTR_OFFSET)); 752 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > fw_len) { 753 device_printf(sc->dev, "Bad firmware file"); 754 status = EIO; 755 goto abort_with_buffer; 756 } 757 hdr = (const void*)(inflate_buffer + hdr_offset); 758 759 status = mxge_validate_firmware(sc, hdr); 760 if (status != 0) 761 goto abort_with_buffer; 762 763 /* Copy the inflated firmware to NIC SRAM. */ 764 for (i = 0; i < fw_len; i += 256) { 765 mxge_pio_copy(sc->sram + MXGE_FW_OFFSET + i, 766 inflate_buffer + i, 767 min(256U, (unsigned)(fw_len - i))); 768 wmb(); 769 dummy = *sc->sram; 770 wmb(); 771 } 772 773 *limit = fw_len; 774 status = 0; 775 abort_with_buffer: 776 free(inflate_buffer, M_TEMP); 777 abort_with_zs: 778 inflateEnd(&zs); 779 abort_with_fw: 780 firmware_put(fw, FIRMWARE_UNLOAD); 781 return status; 782 } 783 784 /* 785 * Enable or disable periodic RDMAs from the host to make certain 786 * chipsets resend dropped PCIe messages 787 */ 788 789 static void 790 mxge_dummy_rdma(mxge_softc_t *sc, int enable) 791 { 792 char buf_bytes[72]; 793 volatile uint32_t *confirm; 794 volatile char *submit; 795 uint32_t *buf, dma_low, dma_high; 796 int i; 797 798 buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL); 799 800 /* clear confirmation addr */ 801 confirm = (volatile uint32_t *)sc->cmd; 802 *confirm = 0; 803 wmb(); 804 805 /* send an rdma command to the PCIe engine, and wait for the 806 response in the confirmation address. The firmware should 807 write a -1 there to indicate it is alive and well 808 */ 809 810 dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr); 811 dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr); 812 buf[0] = htobe32(dma_high); /* confirm addr MSW */ 813 buf[1] = htobe32(dma_low); /* confirm addr LSW */ 814 buf[2] = htobe32(0xffffffff); /* confirm data */ 815 dma_low = MXGE_LOWPART_TO_U32(sc->zeropad_dma.bus_addr); 816 dma_high = MXGE_HIGHPART_TO_U32(sc->zeropad_dma.bus_addr); 817 buf[3] = htobe32(dma_high); /* dummy addr MSW */ 818 buf[4] = htobe32(dma_low); /* dummy addr LSW */ 819 buf[5] = htobe32(enable); /* enable? */ 820 821 822 submit = (volatile char *)(sc->sram + MXGEFW_BOOT_DUMMY_RDMA); 823 824 mxge_pio_copy(submit, buf, 64); 825 wmb(); 826 DELAY(1000); 827 wmb(); 828 i = 0; 829 while (*confirm != 0xffffffff && i < 20) { 830 DELAY(1000); 831 i++; 832 } 833 if (*confirm != 0xffffffff) { 834 device_printf(sc->dev, "dummy rdma %s failed (%p = 0x%x)", 835 (enable ? "enable" : "disable"), confirm, 836 *confirm); 837 } 838 return; 839 } 840 841 static int 842 mxge_send_cmd(mxge_softc_t *sc, uint32_t cmd, mxge_cmd_t *data) 843 { 844 mcp_cmd_t *buf; 845 char buf_bytes[sizeof(*buf) + 8]; 846 volatile mcp_cmd_response_t *response = sc->cmd; 847 volatile char *cmd_addr = sc->sram + MXGEFW_ETH_CMD; 848 uint32_t dma_low, dma_high; 849 int err, sleep_total = 0; 850 851 /* ensure buf is aligned to 8 bytes */ 852 buf = (mcp_cmd_t *)((unsigned long)(buf_bytes + 7) & ~7UL); 853 854 buf->data0 = htobe32(data->data0); 855 buf->data1 = htobe32(data->data1); 856 buf->data2 = htobe32(data->data2); 857 buf->cmd = htobe32(cmd); 858 dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr); 859 dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr); 860 861 buf->response_addr.low = htobe32(dma_low); 862 buf->response_addr.high = htobe32(dma_high); 863 mtx_lock(&sc->cmd_mtx); 864 response->result = 0xffffffff; 865 wmb(); 866 mxge_pio_copy((volatile void *)cmd_addr, buf, sizeof (*buf)); 867 868 /* wait up to 20ms */ 869 err = EAGAIN; 870 for (sleep_total = 0; sleep_total < 20; sleep_total++) { 871 bus_dmamap_sync(sc->cmd_dma.dmat, 872 sc->cmd_dma.map, BUS_DMASYNC_POSTREAD); 873 wmb(); 874 switch (be32toh(response->result)) { 875 case 0: 876 data->data0 = be32toh(response->data); 877 err = 0; 878 break; 879 case 0xffffffff: 880 DELAY(1000); 881 break; 882 case MXGEFW_CMD_UNKNOWN: 883 err = ENOSYS; 884 break; 885 case MXGEFW_CMD_ERROR_UNALIGNED: 886 err = E2BIG; 887 break; 888 case MXGEFW_CMD_ERROR_BUSY: 889 err = EBUSY; 890 break; 891 case MXGEFW_CMD_ERROR_I2C_ABSENT: 892 err = ENXIO; 893 break; 894 default: 895 device_printf(sc->dev, 896 "mxge: command %d " 897 "failed, result = %d\n", 898 cmd, be32toh(response->result)); 899 err = ENXIO; 900 break; 901 } 902 if (err != EAGAIN) 903 break; 904 } 905 if (err == EAGAIN) 906 device_printf(sc->dev, "mxge: command %d timed out" 907 "result = %d\n", 908 cmd, be32toh(response->result)); 909 mtx_unlock(&sc->cmd_mtx); 910 return err; 911 } 912 913 static int 914 mxge_adopt_running_firmware(mxge_softc_t *sc) 915 { 916 struct mcp_gen_header *hdr; 917 const size_t bytes = sizeof (struct mcp_gen_header); 918 size_t hdr_offset; 919 int status; 920 921 /* find running firmware header */ 922 hdr_offset = htobe32(*(volatile uint32_t *) 923 (sc->sram + MCP_HEADER_PTR_OFFSET)); 924 925 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > sc->sram_size) { 926 device_printf(sc->dev, 927 "Running firmware has bad header offset (%d)\n", 928 (int)hdr_offset); 929 return EIO; 930 } 931 932 /* copy header of running firmware from SRAM to host memory to 933 * validate firmware */ 934 hdr = malloc(bytes, M_DEVBUF, M_NOWAIT); 935 if (hdr == NULL) { 936 device_printf(sc->dev, "could not malloc firmware hdr\n"); 937 return ENOMEM; 938 } 939 bus_space_read_region_1(rman_get_bustag(sc->mem_res), 940 rman_get_bushandle(sc->mem_res), 941 hdr_offset, (char *)hdr, bytes); 942 status = mxge_validate_firmware(sc, hdr); 943 free(hdr, M_DEVBUF); 944 945 /* 946 * check to see if adopted firmware has bug where adopting 947 * it will cause broadcasts to be filtered unless the NIC 948 * is kept in ALLMULTI mode 949 */ 950 if (sc->fw_ver_major == 1 && sc->fw_ver_minor == 4 && 951 sc->fw_ver_tiny >= 4 && sc->fw_ver_tiny <= 11) { 952 sc->adopted_rx_filter_bug = 1; 953 device_printf(sc->dev, "Adopting fw %d.%d.%d: " 954 "working around rx filter bug\n", 955 sc->fw_ver_major, sc->fw_ver_minor, 956 sc->fw_ver_tiny); 957 } 958 959 return status; 960 } 961 962 963 static int 964 mxge_load_firmware(mxge_softc_t *sc, int adopt) 965 { 966 volatile uint32_t *confirm; 967 volatile char *submit; 968 char buf_bytes[72]; 969 uint32_t *buf, size, dma_low, dma_high; 970 int status, i; 971 972 buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL); 973 974 size = sc->sram_size; 975 status = mxge_load_firmware_helper(sc, &size); 976 if (status) { 977 if (!adopt) 978 return status; 979 /* Try to use the currently running firmware, if 980 it is new enough */ 981 status = mxge_adopt_running_firmware(sc); 982 if (status) { 983 device_printf(sc->dev, 984 "failed to adopt running firmware\n"); 985 return status; 986 } 987 device_printf(sc->dev, 988 "Successfully adopted running firmware\n"); 989 if (sc->tx_boundary == 4096) { 990 device_printf(sc->dev, 991 "Using firmware currently running on NIC" 992 ". For optimal\n"); 993 device_printf(sc->dev, 994 "performance consider loading optimized " 995 "firmware\n"); 996 } 997 sc->fw_name = mxge_fw_unaligned; 998 sc->tx_boundary = 2048; 999 return 0; 1000 } 1001 /* clear confirmation addr */ 1002 confirm = (volatile uint32_t *)sc->cmd; 1003 *confirm = 0; 1004 wmb(); 1005 /* send a reload command to the bootstrap MCP, and wait for the 1006 response in the confirmation address. The firmware should 1007 write a -1 there to indicate it is alive and well 1008 */ 1009 1010 dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr); 1011 dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr); 1012 1013 buf[0] = htobe32(dma_high); /* confirm addr MSW */ 1014 buf[1] = htobe32(dma_low); /* confirm addr LSW */ 1015 buf[2] = htobe32(0xffffffff); /* confirm data */ 1016 1017 /* FIX: All newest firmware should un-protect the bottom of 1018 the sram before handoff. However, the very first interfaces 1019 do not. Therefore the handoff copy must skip the first 8 bytes 1020 */ 1021 /* where the code starts*/ 1022 buf[3] = htobe32(MXGE_FW_OFFSET + 8); 1023 buf[4] = htobe32(size - 8); /* length of code */ 1024 buf[5] = htobe32(8); /* where to copy to */ 1025 buf[6] = htobe32(0); /* where to jump to */ 1026 1027 submit = (volatile char *)(sc->sram + MXGEFW_BOOT_HANDOFF); 1028 mxge_pio_copy(submit, buf, 64); 1029 wmb(); 1030 DELAY(1000); 1031 wmb(); 1032 i = 0; 1033 while (*confirm != 0xffffffff && i < 20) { 1034 DELAY(1000*10); 1035 i++; 1036 bus_dmamap_sync(sc->cmd_dma.dmat, 1037 sc->cmd_dma.map, BUS_DMASYNC_POSTREAD); 1038 } 1039 if (*confirm != 0xffffffff) { 1040 device_printf(sc->dev,"handoff failed (%p = 0x%x)", 1041 confirm, *confirm); 1042 1043 return ENXIO; 1044 } 1045 return 0; 1046 } 1047 1048 static int 1049 mxge_update_mac_address(mxge_softc_t *sc) 1050 { 1051 mxge_cmd_t cmd; 1052 uint8_t *addr = sc->mac_addr; 1053 int status; 1054 1055 1056 cmd.data0 = ((addr[0] << 24) | (addr[1] << 16) 1057 | (addr[2] << 8) | addr[3]); 1058 1059 cmd.data1 = ((addr[4] << 8) | (addr[5])); 1060 1061 status = mxge_send_cmd(sc, MXGEFW_SET_MAC_ADDRESS, &cmd); 1062 return status; 1063 } 1064 1065 static int 1066 mxge_change_pause(mxge_softc_t *sc, int pause) 1067 { 1068 mxge_cmd_t cmd; 1069 int status; 1070 1071 if (pause) 1072 status = mxge_send_cmd(sc, MXGEFW_ENABLE_FLOW_CONTROL, 1073 &cmd); 1074 else 1075 status = mxge_send_cmd(sc, MXGEFW_DISABLE_FLOW_CONTROL, 1076 &cmd); 1077 1078 if (status) { 1079 device_printf(sc->dev, "Failed to set flow control mode\n"); 1080 return ENXIO; 1081 } 1082 sc->pause = pause; 1083 return 0; 1084 } 1085 1086 static void 1087 mxge_change_promisc(mxge_softc_t *sc, int promisc) 1088 { 1089 mxge_cmd_t cmd; 1090 int status; 1091 1092 if (mxge_always_promisc) 1093 promisc = 1; 1094 1095 if (promisc) 1096 status = mxge_send_cmd(sc, MXGEFW_ENABLE_PROMISC, 1097 &cmd); 1098 else 1099 status = mxge_send_cmd(sc, MXGEFW_DISABLE_PROMISC, 1100 &cmd); 1101 1102 if (status) { 1103 device_printf(sc->dev, "Failed to set promisc mode\n"); 1104 } 1105 } 1106 1107 static void 1108 mxge_set_multicast_list(mxge_softc_t *sc) 1109 { 1110 mxge_cmd_t cmd; 1111 struct ifmultiaddr *ifma; 1112 struct ifnet *ifp = sc->ifp; 1113 int err; 1114 1115 /* This firmware is known to not support multicast */ 1116 if (!sc->fw_multicast_support) 1117 return; 1118 1119 /* Disable multicast filtering while we play with the lists*/ 1120 err = mxge_send_cmd(sc, MXGEFW_ENABLE_ALLMULTI, &cmd); 1121 if (err != 0) { 1122 device_printf(sc->dev, "Failed MXGEFW_ENABLE_ALLMULTI," 1123 " error status: %d\n", err); 1124 return; 1125 } 1126 1127 if (sc->adopted_rx_filter_bug) 1128 return; 1129 1130 if (ifp->if_flags & IFF_ALLMULTI) 1131 /* request to disable multicast filtering, so quit here */ 1132 return; 1133 1134 /* Flush all the filters */ 1135 1136 err = mxge_send_cmd(sc, MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, &cmd); 1137 if (err != 0) { 1138 device_printf(sc->dev, 1139 "Failed MXGEFW_LEAVE_ALL_MULTICAST_GROUPS" 1140 ", error status: %d\n", err); 1141 return; 1142 } 1143 1144 /* Walk the multicast list, and add each address */ 1145 1146 if_maddr_rlock(ifp); 1147 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1148 if (ifma->ifma_addr->sa_family != AF_LINK) 1149 continue; 1150 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 1151 &cmd.data0, 4); 1152 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr) + 4, 1153 &cmd.data1, 2); 1154 cmd.data0 = htonl(cmd.data0); 1155 cmd.data1 = htonl(cmd.data1); 1156 err = mxge_send_cmd(sc, MXGEFW_JOIN_MULTICAST_GROUP, &cmd); 1157 if (err != 0) { 1158 device_printf(sc->dev, "Failed " 1159 "MXGEFW_JOIN_MULTICAST_GROUP, error status:" 1160 "%d\t", err); 1161 /* abort, leaving multicast filtering off */ 1162 if_maddr_runlock(ifp); 1163 return; 1164 } 1165 } 1166 if_maddr_runlock(ifp); 1167 /* Enable multicast filtering */ 1168 err = mxge_send_cmd(sc, MXGEFW_DISABLE_ALLMULTI, &cmd); 1169 if (err != 0) { 1170 device_printf(sc->dev, "Failed MXGEFW_DISABLE_ALLMULTI" 1171 ", error status: %d\n", err); 1172 } 1173 } 1174 1175 static int 1176 mxge_max_mtu(mxge_softc_t *sc) 1177 { 1178 mxge_cmd_t cmd; 1179 int status; 1180 1181 if (MJUMPAGESIZE - MXGEFW_PAD > MXGEFW_MAX_MTU) 1182 return MXGEFW_MAX_MTU - MXGEFW_PAD; 1183 1184 /* try to set nbufs to see if it we can 1185 use virtually contiguous jumbos */ 1186 cmd.data0 = 0; 1187 status = mxge_send_cmd(sc, MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS, 1188 &cmd); 1189 if (status == 0) 1190 return MXGEFW_MAX_MTU - MXGEFW_PAD; 1191 1192 /* otherwise, we're limited to MJUMPAGESIZE */ 1193 return MJUMPAGESIZE - MXGEFW_PAD; 1194 } 1195 1196 static int 1197 mxge_reset(mxge_softc_t *sc, int interrupts_setup) 1198 { 1199 struct mxge_slice_state *ss; 1200 mxge_rx_done_t *rx_done; 1201 volatile uint32_t *irq_claim; 1202 mxge_cmd_t cmd; 1203 int slice, status; 1204 1205 /* try to send a reset command to the card to see if it 1206 is alive */ 1207 memset(&cmd, 0, sizeof (cmd)); 1208 status = mxge_send_cmd(sc, MXGEFW_CMD_RESET, &cmd); 1209 if (status != 0) { 1210 device_printf(sc->dev, "failed reset\n"); 1211 return ENXIO; 1212 } 1213 1214 mxge_dummy_rdma(sc, 1); 1215 1216 1217 /* set the intrq size */ 1218 cmd.data0 = sc->rx_ring_size; 1219 status = mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd); 1220 1221 /* 1222 * Even though we already know how many slices are supported 1223 * via mxge_slice_probe(), MXGEFW_CMD_GET_MAX_RSS_QUEUES 1224 * has magic side effects, and must be called after a reset. 1225 * It must be called prior to calling any RSS related cmds, 1226 * including assigning an interrupt queue for anything but 1227 * slice 0. It must also be called *after* 1228 * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by 1229 * the firmware to compute offsets. 1230 */ 1231 1232 if (sc->num_slices > 1) { 1233 /* ask the maximum number of slices it supports */ 1234 status = mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_RSS_QUEUES, 1235 &cmd); 1236 if (status != 0) { 1237 device_printf(sc->dev, 1238 "failed to get number of slices\n"); 1239 return status; 1240 } 1241 /* 1242 * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior 1243 * to setting up the interrupt queue DMA 1244 */ 1245 cmd.data0 = sc->num_slices; 1246 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 1247 #ifdef IFNET_BUF_RING 1248 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 1249 #endif 1250 status = mxge_send_cmd(sc, MXGEFW_CMD_ENABLE_RSS_QUEUES, 1251 &cmd); 1252 if (status != 0) { 1253 device_printf(sc->dev, 1254 "failed to set number of slices\n"); 1255 return status; 1256 } 1257 } 1258 1259 1260 if (interrupts_setup) { 1261 /* Now exchange information about interrupts */ 1262 for (slice = 0; slice < sc->num_slices; slice++) { 1263 rx_done = &sc->ss[slice].rx_done; 1264 memset(rx_done->entry, 0, sc->rx_ring_size); 1265 cmd.data0 = MXGE_LOWPART_TO_U32(rx_done->dma.bus_addr); 1266 cmd.data1 = MXGE_HIGHPART_TO_U32(rx_done->dma.bus_addr); 1267 cmd.data2 = slice; 1268 status |= mxge_send_cmd(sc, 1269 MXGEFW_CMD_SET_INTRQ_DMA, 1270 &cmd); 1271 } 1272 } 1273 1274 status |= mxge_send_cmd(sc, 1275 MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd); 1276 1277 1278 sc->intr_coal_delay_ptr = (volatile uint32_t *)(sc->sram + cmd.data0); 1279 1280 status |= mxge_send_cmd(sc, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd); 1281 irq_claim = (volatile uint32_t *)(sc->sram + cmd.data0); 1282 1283 1284 status |= mxge_send_cmd(sc, MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET, 1285 &cmd); 1286 sc->irq_deassert = (volatile uint32_t *)(sc->sram + cmd.data0); 1287 if (status != 0) { 1288 device_printf(sc->dev, "failed set interrupt parameters\n"); 1289 return status; 1290 } 1291 1292 1293 *sc->intr_coal_delay_ptr = htobe32(sc->intr_coal_delay); 1294 1295 1296 /* run a DMA benchmark */ 1297 (void) mxge_dma_test(sc, MXGEFW_DMA_TEST); 1298 1299 for (slice = 0; slice < sc->num_slices; slice++) { 1300 ss = &sc->ss[slice]; 1301 1302 ss->irq_claim = irq_claim + (2 * slice); 1303 /* reset mcp/driver shared state back to 0 */ 1304 ss->rx_done.idx = 0; 1305 ss->rx_done.cnt = 0; 1306 ss->tx.req = 0; 1307 ss->tx.done = 0; 1308 ss->tx.pkt_done = 0; 1309 ss->tx.queue_active = 0; 1310 ss->tx.activate = 0; 1311 ss->tx.deactivate = 0; 1312 ss->tx.wake = 0; 1313 ss->tx.defrag = 0; 1314 ss->tx.stall = 0; 1315 ss->rx_big.cnt = 0; 1316 ss->rx_small.cnt = 0; 1317 ss->lc.lro_bad_csum = 0; 1318 ss->lc.lro_queued = 0; 1319 ss->lc.lro_flushed = 0; 1320 if (ss->fw_stats != NULL) { 1321 bzero(ss->fw_stats, sizeof *ss->fw_stats); 1322 } 1323 } 1324 sc->rdma_tags_available = 15; 1325 status = mxge_update_mac_address(sc); 1326 mxge_change_promisc(sc, sc->ifp->if_flags & IFF_PROMISC); 1327 mxge_change_pause(sc, sc->pause); 1328 mxge_set_multicast_list(sc); 1329 if (sc->throttle) { 1330 cmd.data0 = sc->throttle; 1331 if (mxge_send_cmd(sc, MXGEFW_CMD_SET_THROTTLE_FACTOR, 1332 &cmd)) { 1333 device_printf(sc->dev, 1334 "can't enable throttle\n"); 1335 } 1336 } 1337 return status; 1338 } 1339 1340 static int 1341 mxge_change_throttle(SYSCTL_HANDLER_ARGS) 1342 { 1343 mxge_cmd_t cmd; 1344 mxge_softc_t *sc; 1345 int err; 1346 unsigned int throttle; 1347 1348 sc = arg1; 1349 throttle = sc->throttle; 1350 err = sysctl_handle_int(oidp, &throttle, arg2, req); 1351 if (err != 0) { 1352 return err; 1353 } 1354 1355 if (throttle == sc->throttle) 1356 return 0; 1357 1358 if (throttle < MXGE_MIN_THROTTLE || throttle > MXGE_MAX_THROTTLE) 1359 return EINVAL; 1360 1361 mtx_lock(&sc->driver_mtx); 1362 cmd.data0 = throttle; 1363 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_THROTTLE_FACTOR, &cmd); 1364 if (err == 0) 1365 sc->throttle = throttle; 1366 mtx_unlock(&sc->driver_mtx); 1367 return err; 1368 } 1369 1370 static int 1371 mxge_change_intr_coal(SYSCTL_HANDLER_ARGS) 1372 { 1373 mxge_softc_t *sc; 1374 unsigned int intr_coal_delay; 1375 int err; 1376 1377 sc = arg1; 1378 intr_coal_delay = sc->intr_coal_delay; 1379 err = sysctl_handle_int(oidp, &intr_coal_delay, arg2, req); 1380 if (err != 0) { 1381 return err; 1382 } 1383 if (intr_coal_delay == sc->intr_coal_delay) 1384 return 0; 1385 1386 if (intr_coal_delay == 0 || intr_coal_delay > 1000*1000) 1387 return EINVAL; 1388 1389 mtx_lock(&sc->driver_mtx); 1390 *sc->intr_coal_delay_ptr = htobe32(intr_coal_delay); 1391 sc->intr_coal_delay = intr_coal_delay; 1392 1393 mtx_unlock(&sc->driver_mtx); 1394 return err; 1395 } 1396 1397 static int 1398 mxge_change_flow_control(SYSCTL_HANDLER_ARGS) 1399 { 1400 mxge_softc_t *sc; 1401 unsigned int enabled; 1402 int err; 1403 1404 sc = arg1; 1405 enabled = sc->pause; 1406 err = sysctl_handle_int(oidp, &enabled, arg2, req); 1407 if (err != 0) { 1408 return err; 1409 } 1410 if (enabled == sc->pause) 1411 return 0; 1412 1413 mtx_lock(&sc->driver_mtx); 1414 err = mxge_change_pause(sc, enabled); 1415 mtx_unlock(&sc->driver_mtx); 1416 return err; 1417 } 1418 1419 static int 1420 mxge_handle_be32(SYSCTL_HANDLER_ARGS) 1421 { 1422 int err; 1423 1424 if (arg1 == NULL) 1425 return EFAULT; 1426 arg2 = be32toh(*(int *)arg1); 1427 arg1 = NULL; 1428 err = sysctl_handle_int(oidp, arg1, arg2, req); 1429 1430 return err; 1431 } 1432 1433 static void 1434 mxge_rem_sysctls(mxge_softc_t *sc) 1435 { 1436 struct mxge_slice_state *ss; 1437 int slice; 1438 1439 if (sc->slice_sysctl_tree == NULL) 1440 return; 1441 1442 for (slice = 0; slice < sc->num_slices; slice++) { 1443 ss = &sc->ss[slice]; 1444 if (ss == NULL || ss->sysctl_tree == NULL) 1445 continue; 1446 sysctl_ctx_free(&ss->sysctl_ctx); 1447 ss->sysctl_tree = NULL; 1448 } 1449 sysctl_ctx_free(&sc->slice_sysctl_ctx); 1450 sc->slice_sysctl_tree = NULL; 1451 } 1452 1453 static void 1454 mxge_add_sysctls(mxge_softc_t *sc) 1455 { 1456 struct sysctl_ctx_list *ctx; 1457 struct sysctl_oid_list *children; 1458 mcp_irq_data_t *fw; 1459 struct mxge_slice_state *ss; 1460 int slice; 1461 char slice_num[8]; 1462 1463 ctx = device_get_sysctl_ctx(sc->dev); 1464 children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 1465 fw = sc->ss[0].fw_stats; 1466 1467 /* random information */ 1468 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, 1469 "firmware_version", 1470 CTLFLAG_RD, &sc->fw_version, 1471 0, "firmware version"); 1472 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, 1473 "serial_number", 1474 CTLFLAG_RD, &sc->serial_number_string, 1475 0, "serial number"); 1476 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, 1477 "product_code", 1478 CTLFLAG_RD, &sc->product_code_string, 1479 0, "product_code"); 1480 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1481 "pcie_link_width", 1482 CTLFLAG_RD, &sc->link_width, 1483 0, "tx_boundary"); 1484 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1485 "tx_boundary", 1486 CTLFLAG_RD, &sc->tx_boundary, 1487 0, "tx_boundary"); 1488 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1489 "write_combine", 1490 CTLFLAG_RD, &sc->wc, 1491 0, "write combining PIO?"); 1492 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1493 "read_dma_MBs", 1494 CTLFLAG_RD, &sc->read_dma, 1495 0, "DMA Read speed in MB/s"); 1496 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1497 "write_dma_MBs", 1498 CTLFLAG_RD, &sc->write_dma, 1499 0, "DMA Write speed in MB/s"); 1500 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1501 "read_write_dma_MBs", 1502 CTLFLAG_RD, &sc->read_write_dma, 1503 0, "DMA concurrent Read/Write speed in MB/s"); 1504 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1505 "watchdog_resets", 1506 CTLFLAG_RD, &sc->watchdog_resets, 1507 0, "Number of times NIC was reset"); 1508 1509 1510 /* performance related tunables */ 1511 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1512 "intr_coal_delay", 1513 CTLTYPE_INT|CTLFLAG_RW, sc, 1514 0, mxge_change_intr_coal, 1515 "I", "interrupt coalescing delay in usecs"); 1516 1517 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1518 "throttle", 1519 CTLTYPE_INT|CTLFLAG_RW, sc, 1520 0, mxge_change_throttle, 1521 "I", "transmit throttling"); 1522 1523 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1524 "flow_control_enabled", 1525 CTLTYPE_INT|CTLFLAG_RW, sc, 1526 0, mxge_change_flow_control, 1527 "I", "interrupt coalescing delay in usecs"); 1528 1529 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1530 "deassert_wait", 1531 CTLFLAG_RW, &mxge_deassert_wait, 1532 0, "Wait for IRQ line to go low in ihandler"); 1533 1534 /* stats block from firmware is in network byte order. 1535 Need to swap it */ 1536 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1537 "link_up", 1538 CTLTYPE_INT|CTLFLAG_RD, &fw->link_up, 1539 0, mxge_handle_be32, 1540 "I", "link up"); 1541 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1542 "rdma_tags_available", 1543 CTLTYPE_INT|CTLFLAG_RD, &fw->rdma_tags_available, 1544 0, mxge_handle_be32, 1545 "I", "rdma_tags_available"); 1546 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1547 "dropped_bad_crc32", 1548 CTLTYPE_INT|CTLFLAG_RD, 1549 &fw->dropped_bad_crc32, 1550 0, mxge_handle_be32, 1551 "I", "dropped_bad_crc32"); 1552 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1553 "dropped_bad_phy", 1554 CTLTYPE_INT|CTLFLAG_RD, 1555 &fw->dropped_bad_phy, 1556 0, mxge_handle_be32, 1557 "I", "dropped_bad_phy"); 1558 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1559 "dropped_link_error_or_filtered", 1560 CTLTYPE_INT|CTLFLAG_RD, 1561 &fw->dropped_link_error_or_filtered, 1562 0, mxge_handle_be32, 1563 "I", "dropped_link_error_or_filtered"); 1564 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1565 "dropped_link_overflow", 1566 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_link_overflow, 1567 0, mxge_handle_be32, 1568 "I", "dropped_link_overflow"); 1569 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1570 "dropped_multicast_filtered", 1571 CTLTYPE_INT|CTLFLAG_RD, 1572 &fw->dropped_multicast_filtered, 1573 0, mxge_handle_be32, 1574 "I", "dropped_multicast_filtered"); 1575 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1576 "dropped_no_big_buffer", 1577 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_no_big_buffer, 1578 0, mxge_handle_be32, 1579 "I", "dropped_no_big_buffer"); 1580 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1581 "dropped_no_small_buffer", 1582 CTLTYPE_INT|CTLFLAG_RD, 1583 &fw->dropped_no_small_buffer, 1584 0, mxge_handle_be32, 1585 "I", "dropped_no_small_buffer"); 1586 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1587 "dropped_overrun", 1588 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_overrun, 1589 0, mxge_handle_be32, 1590 "I", "dropped_overrun"); 1591 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1592 "dropped_pause", 1593 CTLTYPE_INT|CTLFLAG_RD, 1594 &fw->dropped_pause, 1595 0, mxge_handle_be32, 1596 "I", "dropped_pause"); 1597 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1598 "dropped_runt", 1599 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_runt, 1600 0, mxge_handle_be32, 1601 "I", "dropped_runt"); 1602 1603 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1604 "dropped_unicast_filtered", 1605 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_unicast_filtered, 1606 0, mxge_handle_be32, 1607 "I", "dropped_unicast_filtered"); 1608 1609 /* verbose printing? */ 1610 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1611 "verbose", 1612 CTLFLAG_RW, &mxge_verbose, 1613 0, "verbose printing"); 1614 1615 /* add counters exported for debugging from all slices */ 1616 sysctl_ctx_init(&sc->slice_sysctl_ctx); 1617 sc->slice_sysctl_tree = 1618 SYSCTL_ADD_NODE(&sc->slice_sysctl_ctx, children, OID_AUTO, 1619 "slice", CTLFLAG_RD, 0, ""); 1620 1621 for (slice = 0; slice < sc->num_slices; slice++) { 1622 ss = &sc->ss[slice]; 1623 sysctl_ctx_init(&ss->sysctl_ctx); 1624 ctx = &ss->sysctl_ctx; 1625 children = SYSCTL_CHILDREN(sc->slice_sysctl_tree); 1626 sprintf(slice_num, "%d", slice); 1627 ss->sysctl_tree = 1628 SYSCTL_ADD_NODE(ctx, children, OID_AUTO, slice_num, 1629 CTLFLAG_RD, 0, ""); 1630 children = SYSCTL_CHILDREN(ss->sysctl_tree); 1631 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1632 "rx_small_cnt", 1633 CTLFLAG_RD, &ss->rx_small.cnt, 1634 0, "rx_small_cnt"); 1635 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1636 "rx_big_cnt", 1637 CTLFLAG_RD, &ss->rx_big.cnt, 1638 0, "rx_small_cnt"); 1639 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1640 "lro_flushed", CTLFLAG_RD, &ss->lc.lro_flushed, 1641 0, "number of lro merge queues flushed"); 1642 1643 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1644 "lro_bad_csum", CTLFLAG_RD, &ss->lc.lro_bad_csum, 1645 0, "number of bad csums preventing LRO"); 1646 1647 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1648 "lro_queued", CTLFLAG_RD, &ss->lc.lro_queued, 1649 0, "number of frames appended to lro merge" 1650 "queues"); 1651 1652 #ifndef IFNET_BUF_RING 1653 /* only transmit from slice 0 for now */ 1654 if (slice > 0) 1655 continue; 1656 #endif 1657 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1658 "tx_req", 1659 CTLFLAG_RD, &ss->tx.req, 1660 0, "tx_req"); 1661 1662 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1663 "tx_done", 1664 CTLFLAG_RD, &ss->tx.done, 1665 0, "tx_done"); 1666 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1667 "tx_pkt_done", 1668 CTLFLAG_RD, &ss->tx.pkt_done, 1669 0, "tx_done"); 1670 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1671 "tx_stall", 1672 CTLFLAG_RD, &ss->tx.stall, 1673 0, "tx_stall"); 1674 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1675 "tx_wake", 1676 CTLFLAG_RD, &ss->tx.wake, 1677 0, "tx_wake"); 1678 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1679 "tx_defrag", 1680 CTLFLAG_RD, &ss->tx.defrag, 1681 0, "tx_defrag"); 1682 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1683 "tx_queue_active", 1684 CTLFLAG_RD, &ss->tx.queue_active, 1685 0, "tx_queue_active"); 1686 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1687 "tx_activate", 1688 CTLFLAG_RD, &ss->tx.activate, 1689 0, "tx_activate"); 1690 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1691 "tx_deactivate", 1692 CTLFLAG_RD, &ss->tx.deactivate, 1693 0, "tx_deactivate"); 1694 } 1695 } 1696 1697 /* copy an array of mcp_kreq_ether_send_t's to the mcp. Copy 1698 backwards one at a time and handle ring wraps */ 1699 1700 static inline void 1701 mxge_submit_req_backwards(mxge_tx_ring_t *tx, 1702 mcp_kreq_ether_send_t *src, int cnt) 1703 { 1704 int idx, starting_slot; 1705 starting_slot = tx->req; 1706 while (cnt > 1) { 1707 cnt--; 1708 idx = (starting_slot + cnt) & tx->mask; 1709 mxge_pio_copy(&tx->lanai[idx], 1710 &src[cnt], sizeof(*src)); 1711 wmb(); 1712 } 1713 } 1714 1715 /* 1716 * copy an array of mcp_kreq_ether_send_t's to the mcp. Copy 1717 * at most 32 bytes at a time, so as to avoid involving the software 1718 * pio handler in the nic. We re-write the first segment's flags 1719 * to mark them valid only after writing the entire chain 1720 */ 1721 1722 static inline void 1723 mxge_submit_req(mxge_tx_ring_t *tx, mcp_kreq_ether_send_t *src, 1724 int cnt) 1725 { 1726 int idx, i; 1727 uint32_t *src_ints; 1728 volatile uint32_t *dst_ints; 1729 mcp_kreq_ether_send_t *srcp; 1730 volatile mcp_kreq_ether_send_t *dstp, *dst; 1731 uint8_t last_flags; 1732 1733 idx = tx->req & tx->mask; 1734 1735 last_flags = src->flags; 1736 src->flags = 0; 1737 wmb(); 1738 dst = dstp = &tx->lanai[idx]; 1739 srcp = src; 1740 1741 if ((idx + cnt) < tx->mask) { 1742 for (i = 0; i < (cnt - 1); i += 2) { 1743 mxge_pio_copy(dstp, srcp, 2 * sizeof(*src)); 1744 wmb(); /* force write every 32 bytes */ 1745 srcp += 2; 1746 dstp += 2; 1747 } 1748 } else { 1749 /* submit all but the first request, and ensure 1750 that it is submitted below */ 1751 mxge_submit_req_backwards(tx, src, cnt); 1752 i = 0; 1753 } 1754 if (i < cnt) { 1755 /* submit the first request */ 1756 mxge_pio_copy(dstp, srcp, sizeof(*src)); 1757 wmb(); /* barrier before setting valid flag */ 1758 } 1759 1760 /* re-write the last 32-bits with the valid flags */ 1761 src->flags = last_flags; 1762 src_ints = (uint32_t *)src; 1763 src_ints+=3; 1764 dst_ints = (volatile uint32_t *)dst; 1765 dst_ints+=3; 1766 *dst_ints = *src_ints; 1767 tx->req += cnt; 1768 wmb(); 1769 } 1770 1771 static int 1772 mxge_parse_tx(struct mxge_slice_state *ss, struct mbuf *m, 1773 struct mxge_pkt_info *pi) 1774 { 1775 struct ether_vlan_header *eh; 1776 uint16_t etype; 1777 int tso = m->m_pkthdr.csum_flags & (CSUM_TSO); 1778 #if IFCAP_TSO6 && defined(INET6) 1779 int nxt; 1780 #endif 1781 1782 eh = mtod(m, struct ether_vlan_header *); 1783 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 1784 etype = ntohs(eh->evl_proto); 1785 pi->ip_off = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 1786 } else { 1787 etype = ntohs(eh->evl_encap_proto); 1788 pi->ip_off = ETHER_HDR_LEN; 1789 } 1790 1791 switch (etype) { 1792 case ETHERTYPE_IP: 1793 /* 1794 * ensure ip header is in first mbuf, copy it to a 1795 * scratch buffer if not 1796 */ 1797 pi->ip = (struct ip *)(m->m_data + pi->ip_off); 1798 pi->ip6 = NULL; 1799 if (__predict_false(m->m_len < pi->ip_off + sizeof(*pi->ip))) { 1800 m_copydata(m, 0, pi->ip_off + sizeof(*pi->ip), 1801 ss->scratch); 1802 pi->ip = (struct ip *)(ss->scratch + pi->ip_off); 1803 } 1804 pi->ip_hlen = pi->ip->ip_hl << 2; 1805 if (!tso) 1806 return 0; 1807 1808 if (__predict_false(m->m_len < pi->ip_off + pi->ip_hlen + 1809 sizeof(struct tcphdr))) { 1810 m_copydata(m, 0, pi->ip_off + pi->ip_hlen + 1811 sizeof(struct tcphdr), ss->scratch); 1812 pi->ip = (struct ip *)(ss->scratch + pi->ip_off); 1813 } 1814 pi->tcp = (struct tcphdr *)((char *)pi->ip + pi->ip_hlen); 1815 break; 1816 #if IFCAP_TSO6 && defined(INET6) 1817 case ETHERTYPE_IPV6: 1818 pi->ip6 = (struct ip6_hdr *)(m->m_data + pi->ip_off); 1819 if (__predict_false(m->m_len < pi->ip_off + sizeof(*pi->ip6))) { 1820 m_copydata(m, 0, pi->ip_off + sizeof(*pi->ip6), 1821 ss->scratch); 1822 pi->ip6 = (struct ip6_hdr *)(ss->scratch + pi->ip_off); 1823 } 1824 nxt = 0; 1825 pi->ip_hlen = ip6_lasthdr(m, pi->ip_off, IPPROTO_IPV6, &nxt); 1826 pi->ip_hlen -= pi->ip_off; 1827 if (nxt != IPPROTO_TCP && nxt != IPPROTO_UDP) 1828 return EINVAL; 1829 1830 if (!tso) 1831 return 0; 1832 1833 if (pi->ip_off + pi->ip_hlen > ss->sc->max_tso6_hlen) 1834 return EINVAL; 1835 1836 if (__predict_false(m->m_len < pi->ip_off + pi->ip_hlen + 1837 sizeof(struct tcphdr))) { 1838 m_copydata(m, 0, pi->ip_off + pi->ip_hlen + 1839 sizeof(struct tcphdr), ss->scratch); 1840 pi->ip6 = (struct ip6_hdr *)(ss->scratch + pi->ip_off); 1841 } 1842 pi->tcp = (struct tcphdr *)((char *)pi->ip6 + pi->ip_hlen); 1843 break; 1844 #endif 1845 default: 1846 return EINVAL; 1847 } 1848 return 0; 1849 } 1850 1851 #if IFCAP_TSO4 1852 1853 static void 1854 mxge_encap_tso(struct mxge_slice_state *ss, struct mbuf *m, 1855 int busdma_seg_cnt, struct mxge_pkt_info *pi) 1856 { 1857 mxge_tx_ring_t *tx; 1858 mcp_kreq_ether_send_t *req; 1859 bus_dma_segment_t *seg; 1860 uint32_t low, high_swapped; 1861 int len, seglen, cum_len, cum_len_next; 1862 int next_is_first, chop, cnt, rdma_count, small; 1863 uint16_t pseudo_hdr_offset, cksum_offset, mss, sum; 1864 uint8_t flags, flags_next; 1865 static int once; 1866 1867 mss = m->m_pkthdr.tso_segsz; 1868 1869 /* negative cum_len signifies to the 1870 * send loop that we are still in the 1871 * header portion of the TSO packet. 1872 */ 1873 1874 cksum_offset = pi->ip_off + pi->ip_hlen; 1875 cum_len = -(cksum_offset + (pi->tcp->th_off << 2)); 1876 1877 /* TSO implies checksum offload on this hardware */ 1878 if (__predict_false((m->m_pkthdr.csum_flags & (CSUM_TCP|CSUM_TCP_IPV6)) == 0)) { 1879 /* 1880 * If packet has full TCP csum, replace it with pseudo hdr 1881 * sum that the NIC expects, otherwise the NIC will emit 1882 * packets with bad TCP checksums. 1883 */ 1884 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); 1885 if (pi->ip6) { 1886 #if (CSUM_TCP_IPV6 != 0) && defined(INET6) 1887 m->m_pkthdr.csum_flags |= CSUM_TCP_IPV6; 1888 sum = in6_cksum_pseudo(pi->ip6, 1889 m->m_pkthdr.len - cksum_offset, 1890 IPPROTO_TCP, 0); 1891 #endif 1892 } else { 1893 #ifdef INET 1894 m->m_pkthdr.csum_flags |= CSUM_TCP; 1895 sum = in_pseudo(pi->ip->ip_src.s_addr, 1896 pi->ip->ip_dst.s_addr, 1897 htons(IPPROTO_TCP + (m->m_pkthdr.len - 1898 cksum_offset))); 1899 #endif 1900 } 1901 m_copyback(m, offsetof(struct tcphdr, th_sum) + 1902 cksum_offset, sizeof(sum), (caddr_t)&sum); 1903 } 1904 flags = MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST; 1905 1906 1907 /* for TSO, pseudo_hdr_offset holds mss. 1908 * The firmware figures out where to put 1909 * the checksum by parsing the header. */ 1910 pseudo_hdr_offset = htobe16(mss); 1911 1912 if (pi->ip6) { 1913 /* 1914 * for IPv6 TSO, the "checksum offset" is re-purposed 1915 * to store the TCP header len 1916 */ 1917 cksum_offset = (pi->tcp->th_off << 2); 1918 } 1919 1920 tx = &ss->tx; 1921 req = tx->req_list; 1922 seg = tx->seg_list; 1923 cnt = 0; 1924 rdma_count = 0; 1925 /* "rdma_count" is the number of RDMAs belonging to the 1926 * current packet BEFORE the current send request. For 1927 * non-TSO packets, this is equal to "count". 1928 * For TSO packets, rdma_count needs to be reset 1929 * to 0 after a segment cut. 1930 * 1931 * The rdma_count field of the send request is 1932 * the number of RDMAs of the packet starting at 1933 * that request. For TSO send requests with one ore more cuts 1934 * in the middle, this is the number of RDMAs starting 1935 * after the last cut in the request. All previous 1936 * segments before the last cut implicitly have 1 RDMA. 1937 * 1938 * Since the number of RDMAs is not known beforehand, 1939 * it must be filled-in retroactively - after each 1940 * segmentation cut or at the end of the entire packet. 1941 */ 1942 1943 while (busdma_seg_cnt) { 1944 /* Break the busdma segment up into pieces*/ 1945 low = MXGE_LOWPART_TO_U32(seg->ds_addr); 1946 high_swapped = htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr)); 1947 len = seg->ds_len; 1948 1949 while (len) { 1950 flags_next = flags & ~MXGEFW_FLAGS_FIRST; 1951 seglen = len; 1952 cum_len_next = cum_len + seglen; 1953 (req-rdma_count)->rdma_count = rdma_count + 1; 1954 if (__predict_true(cum_len >= 0)) { 1955 /* payload */ 1956 chop = (cum_len_next > mss); 1957 cum_len_next = cum_len_next % mss; 1958 next_is_first = (cum_len_next == 0); 1959 flags |= chop * MXGEFW_FLAGS_TSO_CHOP; 1960 flags_next |= next_is_first * 1961 MXGEFW_FLAGS_FIRST; 1962 rdma_count |= -(chop | next_is_first); 1963 rdma_count += chop & !next_is_first; 1964 } else if (cum_len_next >= 0) { 1965 /* header ends */ 1966 rdma_count = -1; 1967 cum_len_next = 0; 1968 seglen = -cum_len; 1969 small = (mss <= MXGEFW_SEND_SMALL_SIZE); 1970 flags_next = MXGEFW_FLAGS_TSO_PLD | 1971 MXGEFW_FLAGS_FIRST | 1972 (small * MXGEFW_FLAGS_SMALL); 1973 } 1974 1975 req->addr_high = high_swapped; 1976 req->addr_low = htobe32(low); 1977 req->pseudo_hdr_offset = pseudo_hdr_offset; 1978 req->pad = 0; 1979 req->rdma_count = 1; 1980 req->length = htobe16(seglen); 1981 req->cksum_offset = cksum_offset; 1982 req->flags = flags | ((cum_len & 1) * 1983 MXGEFW_FLAGS_ALIGN_ODD); 1984 low += seglen; 1985 len -= seglen; 1986 cum_len = cum_len_next; 1987 flags = flags_next; 1988 req++; 1989 cnt++; 1990 rdma_count++; 1991 if (cksum_offset != 0 && !pi->ip6) { 1992 if (__predict_false(cksum_offset > seglen)) 1993 cksum_offset -= seglen; 1994 else 1995 cksum_offset = 0; 1996 } 1997 if (__predict_false(cnt > tx->max_desc)) 1998 goto drop; 1999 } 2000 busdma_seg_cnt--; 2001 seg++; 2002 } 2003 (req-rdma_count)->rdma_count = rdma_count; 2004 2005 do { 2006 req--; 2007 req->flags |= MXGEFW_FLAGS_TSO_LAST; 2008 } while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP | MXGEFW_FLAGS_FIRST))); 2009 2010 tx->info[((cnt - 1) + tx->req) & tx->mask].flag = 1; 2011 mxge_submit_req(tx, tx->req_list, cnt); 2012 #ifdef IFNET_BUF_RING 2013 if ((ss->sc->num_slices > 1) && tx->queue_active == 0) { 2014 /* tell the NIC to start polling this slice */ 2015 *tx->send_go = 1; 2016 tx->queue_active = 1; 2017 tx->activate++; 2018 wmb(); 2019 } 2020 #endif 2021 return; 2022 2023 drop: 2024 bus_dmamap_unload(tx->dmat, tx->info[tx->req & tx->mask].map); 2025 m_freem(m); 2026 ss->oerrors++; 2027 if (!once) { 2028 printf("tx->max_desc exceeded via TSO!\n"); 2029 printf("mss = %d, %ld, %d!\n", mss, 2030 (long)seg - (long)tx->seg_list, tx->max_desc); 2031 once = 1; 2032 } 2033 return; 2034 2035 } 2036 2037 #endif /* IFCAP_TSO4 */ 2038 2039 #ifdef MXGE_NEW_VLAN_API 2040 /* 2041 * We reproduce the software vlan tag insertion from 2042 * net/if_vlan.c:vlan_start() here so that we can advertise "hardware" 2043 * vlan tag insertion. We need to advertise this in order to have the 2044 * vlan interface respect our csum offload flags. 2045 */ 2046 static struct mbuf * 2047 mxge_vlan_tag_insert(struct mbuf *m) 2048 { 2049 struct ether_vlan_header *evl; 2050 2051 M_PREPEND(m, ETHER_VLAN_ENCAP_LEN, M_NOWAIT); 2052 if (__predict_false(m == NULL)) 2053 return NULL; 2054 if (m->m_len < sizeof(*evl)) { 2055 m = m_pullup(m, sizeof(*evl)); 2056 if (__predict_false(m == NULL)) 2057 return NULL; 2058 } 2059 /* 2060 * Transform the Ethernet header into an Ethernet header 2061 * with 802.1Q encapsulation. 2062 */ 2063 evl = mtod(m, struct ether_vlan_header *); 2064 bcopy((char *)evl + ETHER_VLAN_ENCAP_LEN, 2065 (char *)evl, ETHER_HDR_LEN - ETHER_TYPE_LEN); 2066 evl->evl_encap_proto = htons(ETHERTYPE_VLAN); 2067 evl->evl_tag = htons(m->m_pkthdr.ether_vtag); 2068 m->m_flags &= ~M_VLANTAG; 2069 return m; 2070 } 2071 #endif /* MXGE_NEW_VLAN_API */ 2072 2073 static void 2074 mxge_encap(struct mxge_slice_state *ss, struct mbuf *m) 2075 { 2076 struct mxge_pkt_info pi = {0,0,0,0}; 2077 mxge_softc_t *sc; 2078 mcp_kreq_ether_send_t *req; 2079 bus_dma_segment_t *seg; 2080 struct mbuf *m_tmp; 2081 struct ifnet *ifp; 2082 mxge_tx_ring_t *tx; 2083 int cnt, cum_len, err, i, idx, odd_flag; 2084 uint16_t pseudo_hdr_offset; 2085 uint8_t flags, cksum_offset; 2086 2087 2088 sc = ss->sc; 2089 ifp = sc->ifp; 2090 tx = &ss->tx; 2091 2092 #ifdef MXGE_NEW_VLAN_API 2093 if (m->m_flags & M_VLANTAG) { 2094 m = mxge_vlan_tag_insert(m); 2095 if (__predict_false(m == NULL)) 2096 goto drop_without_m; 2097 } 2098 #endif 2099 if (m->m_pkthdr.csum_flags & 2100 (CSUM_TSO | CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6)) { 2101 if (mxge_parse_tx(ss, m, &pi)) 2102 goto drop; 2103 } 2104 2105 /* (try to) map the frame for DMA */ 2106 idx = tx->req & tx->mask; 2107 err = bus_dmamap_load_mbuf_sg(tx->dmat, tx->info[idx].map, 2108 m, tx->seg_list, &cnt, 2109 BUS_DMA_NOWAIT); 2110 if (__predict_false(err == EFBIG)) { 2111 /* Too many segments in the chain. Try 2112 to defrag */ 2113 m_tmp = m_defrag(m, M_NOWAIT); 2114 if (m_tmp == NULL) { 2115 goto drop; 2116 } 2117 ss->tx.defrag++; 2118 m = m_tmp; 2119 err = bus_dmamap_load_mbuf_sg(tx->dmat, 2120 tx->info[idx].map, 2121 m, tx->seg_list, &cnt, 2122 BUS_DMA_NOWAIT); 2123 } 2124 if (__predict_false(err != 0)) { 2125 device_printf(sc->dev, "bus_dmamap_load_mbuf_sg returned %d" 2126 " packet len = %d\n", err, m->m_pkthdr.len); 2127 goto drop; 2128 } 2129 bus_dmamap_sync(tx->dmat, tx->info[idx].map, 2130 BUS_DMASYNC_PREWRITE); 2131 tx->info[idx].m = m; 2132 2133 #if IFCAP_TSO4 2134 /* TSO is different enough, we handle it in another routine */ 2135 if (m->m_pkthdr.csum_flags & (CSUM_TSO)) { 2136 mxge_encap_tso(ss, m, cnt, &pi); 2137 return; 2138 } 2139 #endif 2140 2141 req = tx->req_list; 2142 cksum_offset = 0; 2143 pseudo_hdr_offset = 0; 2144 flags = MXGEFW_FLAGS_NO_TSO; 2145 2146 /* checksum offloading? */ 2147 if (m->m_pkthdr.csum_flags & 2148 (CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6)) { 2149 /* ensure ip header is in first mbuf, copy 2150 it to a scratch buffer if not */ 2151 cksum_offset = pi.ip_off + pi.ip_hlen; 2152 pseudo_hdr_offset = cksum_offset + m->m_pkthdr.csum_data; 2153 pseudo_hdr_offset = htobe16(pseudo_hdr_offset); 2154 req->cksum_offset = cksum_offset; 2155 flags |= MXGEFW_FLAGS_CKSUM; 2156 odd_flag = MXGEFW_FLAGS_ALIGN_ODD; 2157 } else { 2158 odd_flag = 0; 2159 } 2160 if (m->m_pkthdr.len < MXGEFW_SEND_SMALL_SIZE) 2161 flags |= MXGEFW_FLAGS_SMALL; 2162 2163 /* convert segments into a request list */ 2164 cum_len = 0; 2165 seg = tx->seg_list; 2166 req->flags = MXGEFW_FLAGS_FIRST; 2167 for (i = 0; i < cnt; i++) { 2168 req->addr_low = 2169 htobe32(MXGE_LOWPART_TO_U32(seg->ds_addr)); 2170 req->addr_high = 2171 htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr)); 2172 req->length = htobe16(seg->ds_len); 2173 req->cksum_offset = cksum_offset; 2174 if (cksum_offset > seg->ds_len) 2175 cksum_offset -= seg->ds_len; 2176 else 2177 cksum_offset = 0; 2178 req->pseudo_hdr_offset = pseudo_hdr_offset; 2179 req->pad = 0; /* complete solid 16-byte block */ 2180 req->rdma_count = 1; 2181 req->flags |= flags | ((cum_len & 1) * odd_flag); 2182 cum_len += seg->ds_len; 2183 seg++; 2184 req++; 2185 req->flags = 0; 2186 } 2187 req--; 2188 /* pad runts to 60 bytes */ 2189 if (cum_len < 60) { 2190 req++; 2191 req->addr_low = 2192 htobe32(MXGE_LOWPART_TO_U32(sc->zeropad_dma.bus_addr)); 2193 req->addr_high = 2194 htobe32(MXGE_HIGHPART_TO_U32(sc->zeropad_dma.bus_addr)); 2195 req->length = htobe16(60 - cum_len); 2196 req->cksum_offset = 0; 2197 req->pseudo_hdr_offset = pseudo_hdr_offset; 2198 req->pad = 0; /* complete solid 16-byte block */ 2199 req->rdma_count = 1; 2200 req->flags |= flags | ((cum_len & 1) * odd_flag); 2201 cnt++; 2202 } 2203 2204 tx->req_list[0].rdma_count = cnt; 2205 #if 0 2206 /* print what the firmware will see */ 2207 for (i = 0; i < cnt; i++) { 2208 printf("%d: addr: 0x%x 0x%x len:%d pso%d," 2209 "cso:%d, flags:0x%x, rdma:%d\n", 2210 i, (int)ntohl(tx->req_list[i].addr_high), 2211 (int)ntohl(tx->req_list[i].addr_low), 2212 (int)ntohs(tx->req_list[i].length), 2213 (int)ntohs(tx->req_list[i].pseudo_hdr_offset), 2214 tx->req_list[i].cksum_offset, tx->req_list[i].flags, 2215 tx->req_list[i].rdma_count); 2216 } 2217 printf("--------------\n"); 2218 #endif 2219 tx->info[((cnt - 1) + tx->req) & tx->mask].flag = 1; 2220 mxge_submit_req(tx, tx->req_list, cnt); 2221 #ifdef IFNET_BUF_RING 2222 if ((ss->sc->num_slices > 1) && tx->queue_active == 0) { 2223 /* tell the NIC to start polling this slice */ 2224 *tx->send_go = 1; 2225 tx->queue_active = 1; 2226 tx->activate++; 2227 wmb(); 2228 } 2229 #endif 2230 return; 2231 2232 drop: 2233 m_freem(m); 2234 drop_without_m: 2235 ss->oerrors++; 2236 return; 2237 } 2238 2239 #ifdef IFNET_BUF_RING 2240 static void 2241 mxge_qflush(struct ifnet *ifp) 2242 { 2243 mxge_softc_t *sc = ifp->if_softc; 2244 mxge_tx_ring_t *tx; 2245 struct mbuf *m; 2246 int slice; 2247 2248 for (slice = 0; slice < sc->num_slices; slice++) { 2249 tx = &sc->ss[slice].tx; 2250 mtx_lock(&tx->mtx); 2251 while ((m = buf_ring_dequeue_sc(tx->br)) != NULL) 2252 m_freem(m); 2253 mtx_unlock(&tx->mtx); 2254 } 2255 if_qflush(ifp); 2256 } 2257 2258 static inline void 2259 mxge_start_locked(struct mxge_slice_state *ss) 2260 { 2261 mxge_softc_t *sc; 2262 struct mbuf *m; 2263 struct ifnet *ifp; 2264 mxge_tx_ring_t *tx; 2265 2266 sc = ss->sc; 2267 ifp = sc->ifp; 2268 tx = &ss->tx; 2269 2270 while ((tx->mask - (tx->req - tx->done)) > tx->max_desc) { 2271 m = drbr_dequeue(ifp, tx->br); 2272 if (m == NULL) { 2273 return; 2274 } 2275 /* let BPF see it */ 2276 BPF_MTAP(ifp, m); 2277 2278 /* give it to the nic */ 2279 mxge_encap(ss, m); 2280 } 2281 /* ran out of transmit slots */ 2282 if (((ss->if_drv_flags & IFF_DRV_OACTIVE) == 0) 2283 && (!drbr_empty(ifp, tx->br))) { 2284 ss->if_drv_flags |= IFF_DRV_OACTIVE; 2285 tx->stall++; 2286 } 2287 } 2288 2289 static int 2290 mxge_transmit_locked(struct mxge_slice_state *ss, struct mbuf *m) 2291 { 2292 mxge_softc_t *sc; 2293 struct ifnet *ifp; 2294 mxge_tx_ring_t *tx; 2295 int err; 2296 2297 sc = ss->sc; 2298 ifp = sc->ifp; 2299 tx = &ss->tx; 2300 2301 if ((ss->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 2302 IFF_DRV_RUNNING) { 2303 err = drbr_enqueue(ifp, tx->br, m); 2304 return (err); 2305 } 2306 2307 if (!drbr_needs_enqueue(ifp, tx->br) && 2308 ((tx->mask - (tx->req - tx->done)) > tx->max_desc)) { 2309 /* let BPF see it */ 2310 BPF_MTAP(ifp, m); 2311 /* give it to the nic */ 2312 mxge_encap(ss, m); 2313 } else if ((err = drbr_enqueue(ifp, tx->br, m)) != 0) { 2314 return (err); 2315 } 2316 if (!drbr_empty(ifp, tx->br)) 2317 mxge_start_locked(ss); 2318 return (0); 2319 } 2320 2321 static int 2322 mxge_transmit(struct ifnet *ifp, struct mbuf *m) 2323 { 2324 mxge_softc_t *sc = ifp->if_softc; 2325 struct mxge_slice_state *ss; 2326 mxge_tx_ring_t *tx; 2327 int err = 0; 2328 int slice; 2329 2330 slice = m->m_pkthdr.flowid; 2331 slice &= (sc->num_slices - 1); /* num_slices always power of 2 */ 2332 2333 ss = &sc->ss[slice]; 2334 tx = &ss->tx; 2335 2336 if (mtx_trylock(&tx->mtx)) { 2337 err = mxge_transmit_locked(ss, m); 2338 mtx_unlock(&tx->mtx); 2339 } else { 2340 err = drbr_enqueue(ifp, tx->br, m); 2341 } 2342 2343 return (err); 2344 } 2345 2346 #else 2347 2348 static inline void 2349 mxge_start_locked(struct mxge_slice_state *ss) 2350 { 2351 mxge_softc_t *sc; 2352 struct mbuf *m; 2353 struct ifnet *ifp; 2354 mxge_tx_ring_t *tx; 2355 2356 sc = ss->sc; 2357 ifp = sc->ifp; 2358 tx = &ss->tx; 2359 while ((tx->mask - (tx->req - tx->done)) > tx->max_desc) { 2360 IFQ_DRV_DEQUEUE(&ifp->if_snd, m); 2361 if (m == NULL) { 2362 return; 2363 } 2364 /* let BPF see it */ 2365 BPF_MTAP(ifp, m); 2366 2367 /* give it to the nic */ 2368 mxge_encap(ss, m); 2369 } 2370 /* ran out of transmit slots */ 2371 if ((sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) { 2372 sc->ifp->if_drv_flags |= IFF_DRV_OACTIVE; 2373 tx->stall++; 2374 } 2375 } 2376 #endif 2377 static void 2378 mxge_start(struct ifnet *ifp) 2379 { 2380 mxge_softc_t *sc = ifp->if_softc; 2381 struct mxge_slice_state *ss; 2382 2383 /* only use the first slice for now */ 2384 ss = &sc->ss[0]; 2385 mtx_lock(&ss->tx.mtx); 2386 mxge_start_locked(ss); 2387 mtx_unlock(&ss->tx.mtx); 2388 } 2389 2390 /* 2391 * copy an array of mcp_kreq_ether_recv_t's to the mcp. Copy 2392 * at most 32 bytes at a time, so as to avoid involving the software 2393 * pio handler in the nic. We re-write the first segment's low 2394 * DMA address to mark it valid only after we write the entire chunk 2395 * in a burst 2396 */ 2397 static inline void 2398 mxge_submit_8rx(volatile mcp_kreq_ether_recv_t *dst, 2399 mcp_kreq_ether_recv_t *src) 2400 { 2401 uint32_t low; 2402 2403 low = src->addr_low; 2404 src->addr_low = 0xffffffff; 2405 mxge_pio_copy(dst, src, 4 * sizeof (*src)); 2406 wmb(); 2407 mxge_pio_copy(dst + 4, src + 4, 4 * sizeof (*src)); 2408 wmb(); 2409 src->addr_low = low; 2410 dst->addr_low = low; 2411 wmb(); 2412 } 2413 2414 static int 2415 mxge_get_buf_small(struct mxge_slice_state *ss, bus_dmamap_t map, int idx) 2416 { 2417 bus_dma_segment_t seg; 2418 struct mbuf *m; 2419 mxge_rx_ring_t *rx = &ss->rx_small; 2420 int cnt, err; 2421 2422 m = m_gethdr(M_NOWAIT, MT_DATA); 2423 if (m == NULL) { 2424 rx->alloc_fail++; 2425 err = ENOBUFS; 2426 goto done; 2427 } 2428 m->m_len = MHLEN; 2429 err = bus_dmamap_load_mbuf_sg(rx->dmat, map, m, 2430 &seg, &cnt, BUS_DMA_NOWAIT); 2431 if (err != 0) { 2432 m_free(m); 2433 goto done; 2434 } 2435 rx->info[idx].m = m; 2436 rx->shadow[idx].addr_low = 2437 htobe32(MXGE_LOWPART_TO_U32(seg.ds_addr)); 2438 rx->shadow[idx].addr_high = 2439 htobe32(MXGE_HIGHPART_TO_U32(seg.ds_addr)); 2440 2441 done: 2442 if ((idx & 7) == 7) 2443 mxge_submit_8rx(&rx->lanai[idx - 7], &rx->shadow[idx - 7]); 2444 return err; 2445 } 2446 2447 static int 2448 mxge_get_buf_big(struct mxge_slice_state *ss, bus_dmamap_t map, int idx) 2449 { 2450 bus_dma_segment_t seg[3]; 2451 struct mbuf *m; 2452 mxge_rx_ring_t *rx = &ss->rx_big; 2453 int cnt, err, i; 2454 2455 m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rx->cl_size); 2456 if (m == NULL) { 2457 rx->alloc_fail++; 2458 err = ENOBUFS; 2459 goto done; 2460 } 2461 m->m_len = rx->mlen; 2462 err = bus_dmamap_load_mbuf_sg(rx->dmat, map, m, 2463 seg, &cnt, BUS_DMA_NOWAIT); 2464 if (err != 0) { 2465 m_free(m); 2466 goto done; 2467 } 2468 rx->info[idx].m = m; 2469 rx->shadow[idx].addr_low = 2470 htobe32(MXGE_LOWPART_TO_U32(seg->ds_addr)); 2471 rx->shadow[idx].addr_high = 2472 htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr)); 2473 2474 #if MXGE_VIRT_JUMBOS 2475 for (i = 1; i < cnt; i++) { 2476 rx->shadow[idx + i].addr_low = 2477 htobe32(MXGE_LOWPART_TO_U32(seg[i].ds_addr)); 2478 rx->shadow[idx + i].addr_high = 2479 htobe32(MXGE_HIGHPART_TO_U32(seg[i].ds_addr)); 2480 } 2481 #endif 2482 2483 done: 2484 for (i = 0; i < rx->nbufs; i++) { 2485 if ((idx & 7) == 7) { 2486 mxge_submit_8rx(&rx->lanai[idx - 7], 2487 &rx->shadow[idx - 7]); 2488 } 2489 idx++; 2490 } 2491 return err; 2492 } 2493 2494 #ifdef INET6 2495 2496 static uint16_t 2497 mxge_csum_generic(uint16_t *raw, int len) 2498 { 2499 uint32_t csum; 2500 2501 2502 csum = 0; 2503 while (len > 0) { 2504 csum += *raw; 2505 raw++; 2506 len -= 2; 2507 } 2508 csum = (csum >> 16) + (csum & 0xffff); 2509 csum = (csum >> 16) + (csum & 0xffff); 2510 return (uint16_t)csum; 2511 } 2512 2513 static inline uint16_t 2514 mxge_rx_csum6(void *p, struct mbuf *m, uint32_t csum) 2515 { 2516 uint32_t partial; 2517 int nxt, cksum_offset; 2518 struct ip6_hdr *ip6 = p; 2519 uint16_t c; 2520 2521 nxt = ip6->ip6_nxt; 2522 cksum_offset = sizeof (*ip6) + ETHER_HDR_LEN; 2523 if (nxt != IPPROTO_TCP && nxt != IPPROTO_UDP) { 2524 cksum_offset = ip6_lasthdr(m, ETHER_HDR_LEN, 2525 IPPROTO_IPV6, &nxt); 2526 if (nxt != IPPROTO_TCP && nxt != IPPROTO_UDP) 2527 return (1); 2528 } 2529 2530 /* 2531 * IPv6 headers do not contain a checksum, and hence 2532 * do not checksum to zero, so they don't "fall out" 2533 * of the partial checksum calculation like IPv4 2534 * headers do. We need to fix the partial checksum by 2535 * subtracting the checksum of the IPv6 header. 2536 */ 2537 2538 partial = mxge_csum_generic((uint16_t *)ip6, cksum_offset - 2539 ETHER_HDR_LEN); 2540 csum += ~partial; 2541 csum += (csum < ~partial); 2542 csum = (csum >> 16) + (csum & 0xFFFF); 2543 csum = (csum >> 16) + (csum & 0xFFFF); 2544 c = in6_cksum_pseudo(ip6, m->m_pkthdr.len - cksum_offset, nxt, 2545 csum); 2546 c ^= 0xffff; 2547 return (c); 2548 } 2549 #endif /* INET6 */ 2550 /* 2551 * Myri10GE hardware checksums are not valid if the sender 2552 * padded the frame with non-zero padding. This is because 2553 * the firmware just does a simple 16-bit 1s complement 2554 * checksum across the entire frame, excluding the first 14 2555 * bytes. It is best to simply to check the checksum and 2556 * tell the stack about it only if the checksum is good 2557 */ 2558 2559 static inline uint16_t 2560 mxge_rx_csum(struct mbuf *m, int csum) 2561 { 2562 struct ether_header *eh; 2563 #ifdef INET 2564 struct ip *ip; 2565 #endif 2566 #if defined(INET) || defined(INET6) 2567 int cap = m->m_pkthdr.rcvif->if_capenable; 2568 #endif 2569 uint16_t c, etype; 2570 2571 2572 eh = mtod(m, struct ether_header *); 2573 etype = ntohs(eh->ether_type); 2574 switch (etype) { 2575 #ifdef INET 2576 case ETHERTYPE_IP: 2577 if ((cap & IFCAP_RXCSUM) == 0) 2578 return (1); 2579 ip = (struct ip *)(eh + 1); 2580 if (ip->ip_p != IPPROTO_TCP && ip->ip_p != IPPROTO_UDP) 2581 return (1); 2582 c = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, 2583 htonl(ntohs(csum) + ntohs(ip->ip_len) - 2584 (ip->ip_hl << 2) + ip->ip_p)); 2585 c ^= 0xffff; 2586 break; 2587 #endif 2588 #ifdef INET6 2589 case ETHERTYPE_IPV6: 2590 if ((cap & IFCAP_RXCSUM_IPV6) == 0) 2591 return (1); 2592 c = mxge_rx_csum6((eh + 1), m, csum); 2593 break; 2594 #endif 2595 default: 2596 c = 1; 2597 } 2598 return (c); 2599 } 2600 2601 static void 2602 mxge_vlan_tag_remove(struct mbuf *m, uint32_t *csum) 2603 { 2604 struct ether_vlan_header *evl; 2605 struct ether_header *eh; 2606 uint32_t partial; 2607 2608 evl = mtod(m, struct ether_vlan_header *); 2609 eh = mtod(m, struct ether_header *); 2610 2611 /* 2612 * fix checksum by subtracting ETHER_VLAN_ENCAP_LEN bytes 2613 * after what the firmware thought was the end of the ethernet 2614 * header. 2615 */ 2616 2617 /* put checksum into host byte order */ 2618 *csum = ntohs(*csum); 2619 partial = ntohl(*(uint32_t *)(mtod(m, char *) + ETHER_HDR_LEN)); 2620 (*csum) += ~partial; 2621 (*csum) += ((*csum) < ~partial); 2622 (*csum) = ((*csum) >> 16) + ((*csum) & 0xFFFF); 2623 (*csum) = ((*csum) >> 16) + ((*csum) & 0xFFFF); 2624 2625 /* restore checksum to network byte order; 2626 later consumers expect this */ 2627 *csum = htons(*csum); 2628 2629 /* save the tag */ 2630 #ifdef MXGE_NEW_VLAN_API 2631 m->m_pkthdr.ether_vtag = ntohs(evl->evl_tag); 2632 #else 2633 { 2634 struct m_tag *mtag; 2635 mtag = m_tag_alloc(MTAG_VLAN, MTAG_VLAN_TAG, sizeof(u_int), 2636 M_NOWAIT); 2637 if (mtag == NULL) 2638 return; 2639 VLAN_TAG_VALUE(mtag) = ntohs(evl->evl_tag); 2640 m_tag_prepend(m, mtag); 2641 } 2642 2643 #endif 2644 m->m_flags |= M_VLANTAG; 2645 2646 /* 2647 * Remove the 802.1q header by copying the Ethernet 2648 * addresses over it and adjusting the beginning of 2649 * the data in the mbuf. The encapsulated Ethernet 2650 * type field is already in place. 2651 */ 2652 bcopy((char *)evl, (char *)evl + ETHER_VLAN_ENCAP_LEN, 2653 ETHER_HDR_LEN - ETHER_TYPE_LEN); 2654 m_adj(m, ETHER_VLAN_ENCAP_LEN); 2655 } 2656 2657 2658 static inline void 2659 mxge_rx_done_big(struct mxge_slice_state *ss, uint32_t len, 2660 uint32_t csum, int lro) 2661 { 2662 mxge_softc_t *sc; 2663 struct ifnet *ifp; 2664 struct mbuf *m; 2665 struct ether_header *eh; 2666 mxge_rx_ring_t *rx; 2667 bus_dmamap_t old_map; 2668 int idx; 2669 2670 sc = ss->sc; 2671 ifp = sc->ifp; 2672 rx = &ss->rx_big; 2673 idx = rx->cnt & rx->mask; 2674 rx->cnt += rx->nbufs; 2675 /* save a pointer to the received mbuf */ 2676 m = rx->info[idx].m; 2677 /* try to replace the received mbuf */ 2678 if (mxge_get_buf_big(ss, rx->extra_map, idx)) { 2679 /* drop the frame -- the old mbuf is re-cycled */ 2680 ifp->if_ierrors++; 2681 return; 2682 } 2683 2684 /* unmap the received buffer */ 2685 old_map = rx->info[idx].map; 2686 bus_dmamap_sync(rx->dmat, old_map, BUS_DMASYNC_POSTREAD); 2687 bus_dmamap_unload(rx->dmat, old_map); 2688 2689 /* swap the bus_dmamap_t's */ 2690 rx->info[idx].map = rx->extra_map; 2691 rx->extra_map = old_map; 2692 2693 /* mcp implicitly skips 1st 2 bytes so that packet is properly 2694 * aligned */ 2695 m->m_data += MXGEFW_PAD; 2696 2697 m->m_pkthdr.rcvif = ifp; 2698 m->m_len = m->m_pkthdr.len = len; 2699 ss->ipackets++; 2700 eh = mtod(m, struct ether_header *); 2701 if (eh->ether_type == htons(ETHERTYPE_VLAN)) { 2702 mxge_vlan_tag_remove(m, &csum); 2703 } 2704 /* if the checksum is valid, mark it in the mbuf header */ 2705 2706 if ((ifp->if_capenable & (IFCAP_RXCSUM_IPV6 | IFCAP_RXCSUM)) && 2707 (0 == mxge_rx_csum(m, csum))) { 2708 /* Tell the stack that the checksum is good */ 2709 m->m_pkthdr.csum_data = 0xffff; 2710 m->m_pkthdr.csum_flags = CSUM_PSEUDO_HDR | 2711 CSUM_DATA_VALID; 2712 2713 #if defined(INET) || defined (INET6) 2714 if (lro && (0 == tcp_lro_rx(&ss->lc, m, 0))) 2715 return; 2716 #endif 2717 } 2718 /* flowid only valid if RSS hashing is enabled */ 2719 if (sc->num_slices > 1) { 2720 m->m_pkthdr.flowid = (ss - sc->ss); 2721 m->m_flags |= M_FLOWID; 2722 } 2723 /* pass the frame up the stack */ 2724 (*ifp->if_input)(ifp, m); 2725 } 2726 2727 static inline void 2728 mxge_rx_done_small(struct mxge_slice_state *ss, uint32_t len, 2729 uint32_t csum, int lro) 2730 { 2731 mxge_softc_t *sc; 2732 struct ifnet *ifp; 2733 struct ether_header *eh; 2734 struct mbuf *m; 2735 mxge_rx_ring_t *rx; 2736 bus_dmamap_t old_map; 2737 int idx; 2738 2739 sc = ss->sc; 2740 ifp = sc->ifp; 2741 rx = &ss->rx_small; 2742 idx = rx->cnt & rx->mask; 2743 rx->cnt++; 2744 /* save a pointer to the received mbuf */ 2745 m = rx->info[idx].m; 2746 /* try to replace the received mbuf */ 2747 if (mxge_get_buf_small(ss, rx->extra_map, idx)) { 2748 /* drop the frame -- the old mbuf is re-cycled */ 2749 ifp->if_ierrors++; 2750 return; 2751 } 2752 2753 /* unmap the received buffer */ 2754 old_map = rx->info[idx].map; 2755 bus_dmamap_sync(rx->dmat, old_map, BUS_DMASYNC_POSTREAD); 2756 bus_dmamap_unload(rx->dmat, old_map); 2757 2758 /* swap the bus_dmamap_t's */ 2759 rx->info[idx].map = rx->extra_map; 2760 rx->extra_map = old_map; 2761 2762 /* mcp implicitly skips 1st 2 bytes so that packet is properly 2763 * aligned */ 2764 m->m_data += MXGEFW_PAD; 2765 2766 m->m_pkthdr.rcvif = ifp; 2767 m->m_len = m->m_pkthdr.len = len; 2768 ss->ipackets++; 2769 eh = mtod(m, struct ether_header *); 2770 if (eh->ether_type == htons(ETHERTYPE_VLAN)) { 2771 mxge_vlan_tag_remove(m, &csum); 2772 } 2773 /* if the checksum is valid, mark it in the mbuf header */ 2774 if ((ifp->if_capenable & (IFCAP_RXCSUM_IPV6 | IFCAP_RXCSUM)) && 2775 (0 == mxge_rx_csum(m, csum))) { 2776 /* Tell the stack that the checksum is good */ 2777 m->m_pkthdr.csum_data = 0xffff; 2778 m->m_pkthdr.csum_flags = CSUM_PSEUDO_HDR | 2779 CSUM_DATA_VALID; 2780 2781 #if defined(INET) || defined (INET6) 2782 if (lro && (0 == tcp_lro_rx(&ss->lc, m, csum))) 2783 return; 2784 #endif 2785 } 2786 /* flowid only valid if RSS hashing is enabled */ 2787 if (sc->num_slices > 1) { 2788 m->m_pkthdr.flowid = (ss - sc->ss); 2789 m->m_flags |= M_FLOWID; 2790 } 2791 /* pass the frame up the stack */ 2792 (*ifp->if_input)(ifp, m); 2793 } 2794 2795 static inline void 2796 mxge_clean_rx_done(struct mxge_slice_state *ss) 2797 { 2798 mxge_rx_done_t *rx_done = &ss->rx_done; 2799 int limit = 0; 2800 uint16_t length; 2801 uint16_t checksum; 2802 int lro; 2803 2804 lro = ss->sc->ifp->if_capenable & IFCAP_LRO; 2805 while (rx_done->entry[rx_done->idx].length != 0) { 2806 length = ntohs(rx_done->entry[rx_done->idx].length); 2807 rx_done->entry[rx_done->idx].length = 0; 2808 checksum = rx_done->entry[rx_done->idx].checksum; 2809 if (length <= (MHLEN - MXGEFW_PAD)) 2810 mxge_rx_done_small(ss, length, checksum, lro); 2811 else 2812 mxge_rx_done_big(ss, length, checksum, lro); 2813 rx_done->cnt++; 2814 rx_done->idx = rx_done->cnt & rx_done->mask; 2815 2816 /* limit potential for livelock */ 2817 if (__predict_false(++limit > rx_done->mask / 2)) 2818 break; 2819 } 2820 #if defined(INET) || defined (INET6) 2821 while (!SLIST_EMPTY(&ss->lc.lro_active)) { 2822 struct lro_entry *lro = SLIST_FIRST(&ss->lc.lro_active); 2823 SLIST_REMOVE_HEAD(&ss->lc.lro_active, next); 2824 tcp_lro_flush(&ss->lc, lro); 2825 } 2826 #endif 2827 } 2828 2829 2830 static inline void 2831 mxge_tx_done(struct mxge_slice_state *ss, uint32_t mcp_idx) 2832 { 2833 struct ifnet *ifp; 2834 mxge_tx_ring_t *tx; 2835 struct mbuf *m; 2836 bus_dmamap_t map; 2837 int idx; 2838 int *flags; 2839 2840 tx = &ss->tx; 2841 ifp = ss->sc->ifp; 2842 while (tx->pkt_done != mcp_idx) { 2843 idx = tx->done & tx->mask; 2844 tx->done++; 2845 m = tx->info[idx].m; 2846 /* mbuf and DMA map only attached to the first 2847 segment per-mbuf */ 2848 if (m != NULL) { 2849 ss->obytes += m->m_pkthdr.len; 2850 if (m->m_flags & M_MCAST) 2851 ss->omcasts++; 2852 ss->opackets++; 2853 tx->info[idx].m = NULL; 2854 map = tx->info[idx].map; 2855 bus_dmamap_unload(tx->dmat, map); 2856 m_freem(m); 2857 } 2858 if (tx->info[idx].flag) { 2859 tx->info[idx].flag = 0; 2860 tx->pkt_done++; 2861 } 2862 } 2863 2864 /* If we have space, clear IFF_OACTIVE to tell the stack that 2865 its OK to send packets */ 2866 #ifdef IFNET_BUF_RING 2867 flags = &ss->if_drv_flags; 2868 #else 2869 flags = &ifp->if_drv_flags; 2870 #endif 2871 mtx_lock(&ss->tx.mtx); 2872 if ((*flags) & IFF_DRV_OACTIVE && 2873 tx->req - tx->done < (tx->mask + 1)/4) { 2874 *(flags) &= ~IFF_DRV_OACTIVE; 2875 ss->tx.wake++; 2876 mxge_start_locked(ss); 2877 } 2878 #ifdef IFNET_BUF_RING 2879 if ((ss->sc->num_slices > 1) && (tx->req == tx->done)) { 2880 /* let the NIC stop polling this queue, since there 2881 * are no more transmits pending */ 2882 if (tx->req == tx->done) { 2883 *tx->send_stop = 1; 2884 tx->queue_active = 0; 2885 tx->deactivate++; 2886 wmb(); 2887 } 2888 } 2889 #endif 2890 mtx_unlock(&ss->tx.mtx); 2891 2892 } 2893 2894 static struct mxge_media_type mxge_xfp_media_types[] = 2895 { 2896 {IFM_10G_CX4, 0x7f, "10GBASE-CX4 (module)"}, 2897 {IFM_10G_SR, (1 << 7), "10GBASE-SR"}, 2898 {IFM_10G_LR, (1 << 6), "10GBASE-LR"}, 2899 {0, (1 << 5), "10GBASE-ER"}, 2900 {IFM_10G_LRM, (1 << 4), "10GBASE-LRM"}, 2901 {0, (1 << 3), "10GBASE-SW"}, 2902 {0, (1 << 2), "10GBASE-LW"}, 2903 {0, (1 << 1), "10GBASE-EW"}, 2904 {0, (1 << 0), "Reserved"} 2905 }; 2906 static struct mxge_media_type mxge_sfp_media_types[] = 2907 { 2908 {IFM_10G_TWINAX, 0, "10GBASE-Twinax"}, 2909 {0, (1 << 7), "Reserved"}, 2910 {IFM_10G_LRM, (1 << 6), "10GBASE-LRM"}, 2911 {IFM_10G_LR, (1 << 5), "10GBASE-LR"}, 2912 {IFM_10G_SR, (1 << 4), "10GBASE-SR"}, 2913 {IFM_10G_TWINAX,(1 << 0), "10GBASE-Twinax"} 2914 }; 2915 2916 static void 2917 mxge_media_set(mxge_softc_t *sc, int media_type) 2918 { 2919 2920 2921 ifmedia_add(&sc->media, IFM_ETHER | IFM_FDX | media_type, 2922 0, NULL); 2923 ifmedia_set(&sc->media, IFM_ETHER | IFM_FDX | media_type); 2924 sc->current_media = media_type; 2925 sc->media.ifm_media = sc->media.ifm_cur->ifm_media; 2926 } 2927 2928 static void 2929 mxge_media_init(mxge_softc_t *sc) 2930 { 2931 char *ptr; 2932 int i; 2933 2934 ifmedia_removeall(&sc->media); 2935 mxge_media_set(sc, IFM_AUTO); 2936 2937 /* 2938 * parse the product code to deterimine the interface type 2939 * (CX4, XFP, Quad Ribbon Fiber) by looking at the character 2940 * after the 3rd dash in the driver's cached copy of the 2941 * EEPROM's product code string. 2942 */ 2943 ptr = sc->product_code_string; 2944 if (ptr == NULL) { 2945 device_printf(sc->dev, "Missing product code\n"); 2946 return; 2947 } 2948 2949 for (i = 0; i < 3; i++, ptr++) { 2950 ptr = strchr(ptr, '-'); 2951 if (ptr == NULL) { 2952 device_printf(sc->dev, 2953 "only %d dashes in PC?!?\n", i); 2954 return; 2955 } 2956 } 2957 if (*ptr == 'C' || *(ptr +1) == 'C') { 2958 /* -C is CX4 */ 2959 sc->connector = MXGE_CX4; 2960 mxge_media_set(sc, IFM_10G_CX4); 2961 } else if (*ptr == 'Q') { 2962 /* -Q is Quad Ribbon Fiber */ 2963 sc->connector = MXGE_QRF; 2964 device_printf(sc->dev, "Quad Ribbon Fiber Media\n"); 2965 /* FreeBSD has no media type for Quad ribbon fiber */ 2966 } else if (*ptr == 'R') { 2967 /* -R is XFP */ 2968 sc->connector = MXGE_XFP; 2969 } else if (*ptr == 'S' || *(ptr +1) == 'S') { 2970 /* -S or -2S is SFP+ */ 2971 sc->connector = MXGE_SFP; 2972 } else { 2973 device_printf(sc->dev, "Unknown media type: %c\n", *ptr); 2974 } 2975 } 2976 2977 /* 2978 * Determine the media type for a NIC. Some XFPs will identify 2979 * themselves only when their link is up, so this is initiated via a 2980 * link up interrupt. However, this can potentially take up to 2981 * several milliseconds, so it is run via the watchdog routine, rather 2982 * than in the interrupt handler itself. 2983 */ 2984 static void 2985 mxge_media_probe(mxge_softc_t *sc) 2986 { 2987 mxge_cmd_t cmd; 2988 char *cage_type; 2989 2990 struct mxge_media_type *mxge_media_types = NULL; 2991 int i, err, ms, mxge_media_type_entries; 2992 uint32_t byte; 2993 2994 sc->need_media_probe = 0; 2995 2996 if (sc->connector == MXGE_XFP) { 2997 /* -R is XFP */ 2998 mxge_media_types = mxge_xfp_media_types; 2999 mxge_media_type_entries = 3000 sizeof (mxge_xfp_media_types) / 3001 sizeof (mxge_xfp_media_types[0]); 3002 byte = MXGE_XFP_COMPLIANCE_BYTE; 3003 cage_type = "XFP"; 3004 } else if (sc->connector == MXGE_SFP) { 3005 /* -S or -2S is SFP+ */ 3006 mxge_media_types = mxge_sfp_media_types; 3007 mxge_media_type_entries = 3008 sizeof (mxge_sfp_media_types) / 3009 sizeof (mxge_sfp_media_types[0]); 3010 cage_type = "SFP+"; 3011 byte = 3; 3012 } else { 3013 /* nothing to do; media type cannot change */ 3014 return; 3015 } 3016 3017 /* 3018 * At this point we know the NIC has an XFP cage, so now we 3019 * try to determine what is in the cage by using the 3020 * firmware's XFP I2C commands to read the XFP 10GbE compilance 3021 * register. We read just one byte, which may take over 3022 * a millisecond 3023 */ 3024 3025 cmd.data0 = 0; /* just fetch 1 byte, not all 256 */ 3026 cmd.data1 = byte; 3027 err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_READ, &cmd); 3028 if (err == MXGEFW_CMD_ERROR_I2C_FAILURE) { 3029 device_printf(sc->dev, "failed to read XFP\n"); 3030 } 3031 if (err == MXGEFW_CMD_ERROR_I2C_ABSENT) { 3032 device_printf(sc->dev, "Type R/S with no XFP!?!?\n"); 3033 } 3034 if (err != MXGEFW_CMD_OK) { 3035 return; 3036 } 3037 3038 /* now we wait for the data to be cached */ 3039 cmd.data0 = byte; 3040 err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_BYTE, &cmd); 3041 for (ms = 0; (err == EBUSY) && (ms < 50); ms++) { 3042 DELAY(1000); 3043 cmd.data0 = byte; 3044 err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_BYTE, &cmd); 3045 } 3046 if (err != MXGEFW_CMD_OK) { 3047 device_printf(sc->dev, "failed to read %s (%d, %dms)\n", 3048 cage_type, err, ms); 3049 return; 3050 } 3051 3052 if (cmd.data0 == mxge_media_types[0].bitmask) { 3053 if (mxge_verbose) 3054 device_printf(sc->dev, "%s:%s\n", cage_type, 3055 mxge_media_types[0].name); 3056 if (sc->current_media != mxge_media_types[0].flag) { 3057 mxge_media_init(sc); 3058 mxge_media_set(sc, mxge_media_types[0].flag); 3059 } 3060 return; 3061 } 3062 for (i = 1; i < mxge_media_type_entries; i++) { 3063 if (cmd.data0 & mxge_media_types[i].bitmask) { 3064 if (mxge_verbose) 3065 device_printf(sc->dev, "%s:%s\n", 3066 cage_type, 3067 mxge_media_types[i].name); 3068 3069 if (sc->current_media != mxge_media_types[i].flag) { 3070 mxge_media_init(sc); 3071 mxge_media_set(sc, mxge_media_types[i].flag); 3072 } 3073 return; 3074 } 3075 } 3076 if (mxge_verbose) 3077 device_printf(sc->dev, "%s media 0x%x unknown\n", 3078 cage_type, cmd.data0); 3079 3080 return; 3081 } 3082 3083 static void 3084 mxge_intr(void *arg) 3085 { 3086 struct mxge_slice_state *ss = arg; 3087 mxge_softc_t *sc = ss->sc; 3088 mcp_irq_data_t *stats = ss->fw_stats; 3089 mxge_tx_ring_t *tx = &ss->tx; 3090 mxge_rx_done_t *rx_done = &ss->rx_done; 3091 uint32_t send_done_count; 3092 uint8_t valid; 3093 3094 3095 #ifndef IFNET_BUF_RING 3096 /* an interrupt on a non-zero slice is implicitly valid 3097 since MSI-X irqs are not shared */ 3098 if (ss != sc->ss) { 3099 mxge_clean_rx_done(ss); 3100 *ss->irq_claim = be32toh(3); 3101 return; 3102 } 3103 #endif 3104 3105 /* make sure the DMA has finished */ 3106 if (!stats->valid) { 3107 return; 3108 } 3109 valid = stats->valid; 3110 3111 if (sc->legacy_irq) { 3112 /* lower legacy IRQ */ 3113 *sc->irq_deassert = 0; 3114 if (!mxge_deassert_wait) 3115 /* don't wait for conf. that irq is low */ 3116 stats->valid = 0; 3117 } else { 3118 stats->valid = 0; 3119 } 3120 3121 /* loop while waiting for legacy irq deassertion */ 3122 do { 3123 /* check for transmit completes and receives */ 3124 send_done_count = be32toh(stats->send_done_count); 3125 while ((send_done_count != tx->pkt_done) || 3126 (rx_done->entry[rx_done->idx].length != 0)) { 3127 if (send_done_count != tx->pkt_done) 3128 mxge_tx_done(ss, (int)send_done_count); 3129 mxge_clean_rx_done(ss); 3130 send_done_count = be32toh(stats->send_done_count); 3131 } 3132 if (sc->legacy_irq && mxge_deassert_wait) 3133 wmb(); 3134 } while (*((volatile uint8_t *) &stats->valid)); 3135 3136 /* fw link & error stats meaningful only on the first slice */ 3137 if (__predict_false((ss == sc->ss) && stats->stats_updated)) { 3138 if (sc->link_state != stats->link_up) { 3139 sc->link_state = stats->link_up; 3140 if (sc->link_state) { 3141 if_link_state_change(sc->ifp, LINK_STATE_UP); 3142 if_initbaudrate(sc->ifp, IF_Gbps(10)); 3143 if (mxge_verbose) 3144 device_printf(sc->dev, "link up\n"); 3145 } else { 3146 if_link_state_change(sc->ifp, LINK_STATE_DOWN); 3147 sc->ifp->if_baudrate = 0; 3148 if (mxge_verbose) 3149 device_printf(sc->dev, "link down\n"); 3150 } 3151 sc->need_media_probe = 1; 3152 } 3153 if (sc->rdma_tags_available != 3154 be32toh(stats->rdma_tags_available)) { 3155 sc->rdma_tags_available = 3156 be32toh(stats->rdma_tags_available); 3157 device_printf(sc->dev, "RDMA timed out! %d tags " 3158 "left\n", sc->rdma_tags_available); 3159 } 3160 3161 if (stats->link_down) { 3162 sc->down_cnt += stats->link_down; 3163 sc->link_state = 0; 3164 if_link_state_change(sc->ifp, LINK_STATE_DOWN); 3165 } 3166 } 3167 3168 /* check to see if we have rx token to pass back */ 3169 if (valid & 0x1) 3170 *ss->irq_claim = be32toh(3); 3171 *(ss->irq_claim + 1) = be32toh(3); 3172 } 3173 3174 static void 3175 mxge_init(void *arg) 3176 { 3177 mxge_softc_t *sc = arg; 3178 struct ifnet *ifp = sc->ifp; 3179 3180 3181 mtx_lock(&sc->driver_mtx); 3182 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 3183 (void) mxge_open(sc); 3184 mtx_unlock(&sc->driver_mtx); 3185 } 3186 3187 3188 3189 static void 3190 mxge_free_slice_mbufs(struct mxge_slice_state *ss) 3191 { 3192 int i; 3193 3194 #if defined(INET) || defined(INET6) 3195 tcp_lro_free(&ss->lc); 3196 #endif 3197 for (i = 0; i <= ss->rx_big.mask; i++) { 3198 if (ss->rx_big.info[i].m == NULL) 3199 continue; 3200 bus_dmamap_unload(ss->rx_big.dmat, 3201 ss->rx_big.info[i].map); 3202 m_freem(ss->rx_big.info[i].m); 3203 ss->rx_big.info[i].m = NULL; 3204 } 3205 3206 for (i = 0; i <= ss->rx_small.mask; i++) { 3207 if (ss->rx_small.info[i].m == NULL) 3208 continue; 3209 bus_dmamap_unload(ss->rx_small.dmat, 3210 ss->rx_small.info[i].map); 3211 m_freem(ss->rx_small.info[i].m); 3212 ss->rx_small.info[i].m = NULL; 3213 } 3214 3215 /* transmit ring used only on the first slice */ 3216 if (ss->tx.info == NULL) 3217 return; 3218 3219 for (i = 0; i <= ss->tx.mask; i++) { 3220 ss->tx.info[i].flag = 0; 3221 if (ss->tx.info[i].m == NULL) 3222 continue; 3223 bus_dmamap_unload(ss->tx.dmat, 3224 ss->tx.info[i].map); 3225 m_freem(ss->tx.info[i].m); 3226 ss->tx.info[i].m = NULL; 3227 } 3228 } 3229 3230 static void 3231 mxge_free_mbufs(mxge_softc_t *sc) 3232 { 3233 int slice; 3234 3235 for (slice = 0; slice < sc->num_slices; slice++) 3236 mxge_free_slice_mbufs(&sc->ss[slice]); 3237 } 3238 3239 static void 3240 mxge_free_slice_rings(struct mxge_slice_state *ss) 3241 { 3242 int i; 3243 3244 3245 if (ss->rx_done.entry != NULL) 3246 mxge_dma_free(&ss->rx_done.dma); 3247 ss->rx_done.entry = NULL; 3248 3249 if (ss->tx.req_bytes != NULL) 3250 free(ss->tx.req_bytes, M_DEVBUF); 3251 ss->tx.req_bytes = NULL; 3252 3253 if (ss->tx.seg_list != NULL) 3254 free(ss->tx.seg_list, M_DEVBUF); 3255 ss->tx.seg_list = NULL; 3256 3257 if (ss->rx_small.shadow != NULL) 3258 free(ss->rx_small.shadow, M_DEVBUF); 3259 ss->rx_small.shadow = NULL; 3260 3261 if (ss->rx_big.shadow != NULL) 3262 free(ss->rx_big.shadow, M_DEVBUF); 3263 ss->rx_big.shadow = NULL; 3264 3265 if (ss->tx.info != NULL) { 3266 if (ss->tx.dmat != NULL) { 3267 for (i = 0; i <= ss->tx.mask; i++) { 3268 bus_dmamap_destroy(ss->tx.dmat, 3269 ss->tx.info[i].map); 3270 } 3271 bus_dma_tag_destroy(ss->tx.dmat); 3272 } 3273 free(ss->tx.info, M_DEVBUF); 3274 } 3275 ss->tx.info = NULL; 3276 3277 if (ss->rx_small.info != NULL) { 3278 if (ss->rx_small.dmat != NULL) { 3279 for (i = 0; i <= ss->rx_small.mask; i++) { 3280 bus_dmamap_destroy(ss->rx_small.dmat, 3281 ss->rx_small.info[i].map); 3282 } 3283 bus_dmamap_destroy(ss->rx_small.dmat, 3284 ss->rx_small.extra_map); 3285 bus_dma_tag_destroy(ss->rx_small.dmat); 3286 } 3287 free(ss->rx_small.info, M_DEVBUF); 3288 } 3289 ss->rx_small.info = NULL; 3290 3291 if (ss->rx_big.info != NULL) { 3292 if (ss->rx_big.dmat != NULL) { 3293 for (i = 0; i <= ss->rx_big.mask; i++) { 3294 bus_dmamap_destroy(ss->rx_big.dmat, 3295 ss->rx_big.info[i].map); 3296 } 3297 bus_dmamap_destroy(ss->rx_big.dmat, 3298 ss->rx_big.extra_map); 3299 bus_dma_tag_destroy(ss->rx_big.dmat); 3300 } 3301 free(ss->rx_big.info, M_DEVBUF); 3302 } 3303 ss->rx_big.info = NULL; 3304 } 3305 3306 static void 3307 mxge_free_rings(mxge_softc_t *sc) 3308 { 3309 int slice; 3310 3311 for (slice = 0; slice < sc->num_slices; slice++) 3312 mxge_free_slice_rings(&sc->ss[slice]); 3313 } 3314 3315 static int 3316 mxge_alloc_slice_rings(struct mxge_slice_state *ss, int rx_ring_entries, 3317 int tx_ring_entries) 3318 { 3319 mxge_softc_t *sc = ss->sc; 3320 size_t bytes; 3321 int err, i; 3322 3323 /* allocate per-slice receive resources */ 3324 3325 ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1; 3326 ss->rx_done.mask = (2 * rx_ring_entries) - 1; 3327 3328 /* allocate the rx shadow rings */ 3329 bytes = rx_ring_entries * sizeof (*ss->rx_small.shadow); 3330 ss->rx_small.shadow = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK); 3331 3332 bytes = rx_ring_entries * sizeof (*ss->rx_big.shadow); 3333 ss->rx_big.shadow = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK); 3334 3335 /* allocate the rx host info rings */ 3336 bytes = rx_ring_entries * sizeof (*ss->rx_small.info); 3337 ss->rx_small.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK); 3338 3339 bytes = rx_ring_entries * sizeof (*ss->rx_big.info); 3340 ss->rx_big.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK); 3341 3342 /* allocate the rx busdma resources */ 3343 err = bus_dma_tag_create(sc->parent_dmat, /* parent */ 3344 1, /* alignment */ 3345 4096, /* boundary */ 3346 BUS_SPACE_MAXADDR, /* low */ 3347 BUS_SPACE_MAXADDR, /* high */ 3348 NULL, NULL, /* filter */ 3349 MHLEN, /* maxsize */ 3350 1, /* num segs */ 3351 MHLEN, /* maxsegsize */ 3352 BUS_DMA_ALLOCNOW, /* flags */ 3353 NULL, NULL, /* lock */ 3354 &ss->rx_small.dmat); /* tag */ 3355 if (err != 0) { 3356 device_printf(sc->dev, "Err %d allocating rx_small dmat\n", 3357 err); 3358 return err; 3359 } 3360 3361 err = bus_dma_tag_create(sc->parent_dmat, /* parent */ 3362 1, /* alignment */ 3363 #if MXGE_VIRT_JUMBOS 3364 4096, /* boundary */ 3365 #else 3366 0, /* boundary */ 3367 #endif 3368 BUS_SPACE_MAXADDR, /* low */ 3369 BUS_SPACE_MAXADDR, /* high */ 3370 NULL, NULL, /* filter */ 3371 3*4096, /* maxsize */ 3372 #if MXGE_VIRT_JUMBOS 3373 3, /* num segs */ 3374 4096, /* maxsegsize*/ 3375 #else 3376 1, /* num segs */ 3377 MJUM9BYTES, /* maxsegsize*/ 3378 #endif 3379 BUS_DMA_ALLOCNOW, /* flags */ 3380 NULL, NULL, /* lock */ 3381 &ss->rx_big.dmat); /* tag */ 3382 if (err != 0) { 3383 device_printf(sc->dev, "Err %d allocating rx_big dmat\n", 3384 err); 3385 return err; 3386 } 3387 for (i = 0; i <= ss->rx_small.mask; i++) { 3388 err = bus_dmamap_create(ss->rx_small.dmat, 0, 3389 &ss->rx_small.info[i].map); 3390 if (err != 0) { 3391 device_printf(sc->dev, "Err %d rx_small dmamap\n", 3392 err); 3393 return err; 3394 } 3395 } 3396 err = bus_dmamap_create(ss->rx_small.dmat, 0, 3397 &ss->rx_small.extra_map); 3398 if (err != 0) { 3399 device_printf(sc->dev, "Err %d extra rx_small dmamap\n", 3400 err); 3401 return err; 3402 } 3403 3404 for (i = 0; i <= ss->rx_big.mask; i++) { 3405 err = bus_dmamap_create(ss->rx_big.dmat, 0, 3406 &ss->rx_big.info[i].map); 3407 if (err != 0) { 3408 device_printf(sc->dev, "Err %d rx_big dmamap\n", 3409 err); 3410 return err; 3411 } 3412 } 3413 err = bus_dmamap_create(ss->rx_big.dmat, 0, 3414 &ss->rx_big.extra_map); 3415 if (err != 0) { 3416 device_printf(sc->dev, "Err %d extra rx_big dmamap\n", 3417 err); 3418 return err; 3419 } 3420 3421 /* now allocate TX resources */ 3422 3423 #ifndef IFNET_BUF_RING 3424 /* only use a single TX ring for now */ 3425 if (ss != ss->sc->ss) 3426 return 0; 3427 #endif 3428 3429 ss->tx.mask = tx_ring_entries - 1; 3430 ss->tx.max_desc = MIN(MXGE_MAX_SEND_DESC, tx_ring_entries / 4); 3431 3432 3433 /* allocate the tx request copy block */ 3434 bytes = 8 + 3435 sizeof (*ss->tx.req_list) * (ss->tx.max_desc + 4); 3436 ss->tx.req_bytes = malloc(bytes, M_DEVBUF, M_WAITOK); 3437 /* ensure req_list entries are aligned to 8 bytes */ 3438 ss->tx.req_list = (mcp_kreq_ether_send_t *) 3439 ((unsigned long)(ss->tx.req_bytes + 7) & ~7UL); 3440 3441 /* allocate the tx busdma segment list */ 3442 bytes = sizeof (*ss->tx.seg_list) * ss->tx.max_desc; 3443 ss->tx.seg_list = (bus_dma_segment_t *) 3444 malloc(bytes, M_DEVBUF, M_WAITOK); 3445 3446 /* allocate the tx host info ring */ 3447 bytes = tx_ring_entries * sizeof (*ss->tx.info); 3448 ss->tx.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK); 3449 3450 /* allocate the tx busdma resources */ 3451 err = bus_dma_tag_create(sc->parent_dmat, /* parent */ 3452 1, /* alignment */ 3453 sc->tx_boundary, /* boundary */ 3454 BUS_SPACE_MAXADDR, /* low */ 3455 BUS_SPACE_MAXADDR, /* high */ 3456 NULL, NULL, /* filter */ 3457 65536 + 256, /* maxsize */ 3458 ss->tx.max_desc - 2, /* num segs */ 3459 sc->tx_boundary, /* maxsegsz */ 3460 BUS_DMA_ALLOCNOW, /* flags */ 3461 NULL, NULL, /* lock */ 3462 &ss->tx.dmat); /* tag */ 3463 3464 if (err != 0) { 3465 device_printf(sc->dev, "Err %d allocating tx dmat\n", 3466 err); 3467 return err; 3468 } 3469 3470 /* now use these tags to setup dmamaps for each slot 3471 in the ring */ 3472 for (i = 0; i <= ss->tx.mask; i++) { 3473 err = bus_dmamap_create(ss->tx.dmat, 0, 3474 &ss->tx.info[i].map); 3475 if (err != 0) { 3476 device_printf(sc->dev, "Err %d tx dmamap\n", 3477 err); 3478 return err; 3479 } 3480 } 3481 return 0; 3482 3483 } 3484 3485 static int 3486 mxge_alloc_rings(mxge_softc_t *sc) 3487 { 3488 mxge_cmd_t cmd; 3489 int tx_ring_size; 3490 int tx_ring_entries, rx_ring_entries; 3491 int err, slice; 3492 3493 /* get ring sizes */ 3494 err = mxge_send_cmd(sc, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd); 3495 tx_ring_size = cmd.data0; 3496 if (err != 0) { 3497 device_printf(sc->dev, "Cannot determine tx ring sizes\n"); 3498 goto abort; 3499 } 3500 3501 tx_ring_entries = tx_ring_size / sizeof (mcp_kreq_ether_send_t); 3502 rx_ring_entries = sc->rx_ring_size / sizeof (mcp_dma_addr_t); 3503 IFQ_SET_MAXLEN(&sc->ifp->if_snd, tx_ring_entries - 1); 3504 sc->ifp->if_snd.ifq_drv_maxlen = sc->ifp->if_snd.ifq_maxlen; 3505 IFQ_SET_READY(&sc->ifp->if_snd); 3506 3507 for (slice = 0; slice < sc->num_slices; slice++) { 3508 err = mxge_alloc_slice_rings(&sc->ss[slice], 3509 rx_ring_entries, 3510 tx_ring_entries); 3511 if (err != 0) 3512 goto abort; 3513 } 3514 return 0; 3515 3516 abort: 3517 mxge_free_rings(sc); 3518 return err; 3519 3520 } 3521 3522 3523 static void 3524 mxge_choose_params(int mtu, int *big_buf_size, int *cl_size, int *nbufs) 3525 { 3526 int bufsize = mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + MXGEFW_PAD; 3527 3528 if (bufsize < MCLBYTES) { 3529 /* easy, everything fits in a single buffer */ 3530 *big_buf_size = MCLBYTES; 3531 *cl_size = MCLBYTES; 3532 *nbufs = 1; 3533 return; 3534 } 3535 3536 if (bufsize < MJUMPAGESIZE) { 3537 /* still easy, everything still fits in a single buffer */ 3538 *big_buf_size = MJUMPAGESIZE; 3539 *cl_size = MJUMPAGESIZE; 3540 *nbufs = 1; 3541 return; 3542 } 3543 #if MXGE_VIRT_JUMBOS 3544 /* now we need to use virtually contiguous buffers */ 3545 *cl_size = MJUM9BYTES; 3546 *big_buf_size = 4096; 3547 *nbufs = mtu / 4096 + 1; 3548 /* needs to be a power of two, so round up */ 3549 if (*nbufs == 3) 3550 *nbufs = 4; 3551 #else 3552 *cl_size = MJUM9BYTES; 3553 *big_buf_size = MJUM9BYTES; 3554 *nbufs = 1; 3555 #endif 3556 } 3557 3558 static int 3559 mxge_slice_open(struct mxge_slice_state *ss, int nbufs, int cl_size) 3560 { 3561 mxge_softc_t *sc; 3562 mxge_cmd_t cmd; 3563 bus_dmamap_t map; 3564 int err, i, slice; 3565 3566 3567 sc = ss->sc; 3568 slice = ss - sc->ss; 3569 3570 #if defined(INET) || defined(INET6) 3571 (void)tcp_lro_init(&ss->lc); 3572 #endif 3573 ss->lc.ifp = sc->ifp; 3574 3575 /* get the lanai pointers to the send and receive rings */ 3576 3577 err = 0; 3578 #ifndef IFNET_BUF_RING 3579 /* We currently only send from the first slice */ 3580 if (slice == 0) { 3581 #endif 3582 cmd.data0 = slice; 3583 err = mxge_send_cmd(sc, MXGEFW_CMD_GET_SEND_OFFSET, &cmd); 3584 ss->tx.lanai = 3585 (volatile mcp_kreq_ether_send_t *)(sc->sram + cmd.data0); 3586 ss->tx.send_go = (volatile uint32_t *) 3587 (sc->sram + MXGEFW_ETH_SEND_GO + 64 * slice); 3588 ss->tx.send_stop = (volatile uint32_t *) 3589 (sc->sram + MXGEFW_ETH_SEND_STOP + 64 * slice); 3590 #ifndef IFNET_BUF_RING 3591 } 3592 #endif 3593 cmd.data0 = slice; 3594 err |= mxge_send_cmd(sc, 3595 MXGEFW_CMD_GET_SMALL_RX_OFFSET, &cmd); 3596 ss->rx_small.lanai = 3597 (volatile mcp_kreq_ether_recv_t *)(sc->sram + cmd.data0); 3598 cmd.data0 = slice; 3599 err |= mxge_send_cmd(sc, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd); 3600 ss->rx_big.lanai = 3601 (volatile mcp_kreq_ether_recv_t *)(sc->sram + cmd.data0); 3602 3603 if (err != 0) { 3604 device_printf(sc->dev, 3605 "failed to get ring sizes or locations\n"); 3606 return EIO; 3607 } 3608 3609 /* stock receive rings */ 3610 for (i = 0; i <= ss->rx_small.mask; i++) { 3611 map = ss->rx_small.info[i].map; 3612 err = mxge_get_buf_small(ss, map, i); 3613 if (err) { 3614 device_printf(sc->dev, "alloced %d/%d smalls\n", 3615 i, ss->rx_small.mask + 1); 3616 return ENOMEM; 3617 } 3618 } 3619 for (i = 0; i <= ss->rx_big.mask; i++) { 3620 ss->rx_big.shadow[i].addr_low = 0xffffffff; 3621 ss->rx_big.shadow[i].addr_high = 0xffffffff; 3622 } 3623 ss->rx_big.nbufs = nbufs; 3624 ss->rx_big.cl_size = cl_size; 3625 ss->rx_big.mlen = ss->sc->ifp->if_mtu + ETHER_HDR_LEN + 3626 ETHER_VLAN_ENCAP_LEN + MXGEFW_PAD; 3627 for (i = 0; i <= ss->rx_big.mask; i += ss->rx_big.nbufs) { 3628 map = ss->rx_big.info[i].map; 3629 err = mxge_get_buf_big(ss, map, i); 3630 if (err) { 3631 device_printf(sc->dev, "alloced %d/%d bigs\n", 3632 i, ss->rx_big.mask + 1); 3633 return ENOMEM; 3634 } 3635 } 3636 return 0; 3637 } 3638 3639 static int 3640 mxge_open(mxge_softc_t *sc) 3641 { 3642 mxge_cmd_t cmd; 3643 int err, big_bytes, nbufs, slice, cl_size, i; 3644 bus_addr_t bus; 3645 volatile uint8_t *itable; 3646 struct mxge_slice_state *ss; 3647 3648 /* Copy the MAC address in case it was overridden */ 3649 bcopy(IF_LLADDR(sc->ifp), sc->mac_addr, ETHER_ADDR_LEN); 3650 3651 err = mxge_reset(sc, 1); 3652 if (err != 0) { 3653 device_printf(sc->dev, "failed to reset\n"); 3654 return EIO; 3655 } 3656 3657 if (sc->num_slices > 1) { 3658 /* setup the indirection table */ 3659 cmd.data0 = sc->num_slices; 3660 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_RSS_TABLE_SIZE, 3661 &cmd); 3662 3663 err |= mxge_send_cmd(sc, MXGEFW_CMD_GET_RSS_TABLE_OFFSET, 3664 &cmd); 3665 if (err != 0) { 3666 device_printf(sc->dev, 3667 "failed to setup rss tables\n"); 3668 return err; 3669 } 3670 3671 /* just enable an identity mapping */ 3672 itable = sc->sram + cmd.data0; 3673 for (i = 0; i < sc->num_slices; i++) 3674 itable[i] = (uint8_t)i; 3675 3676 cmd.data0 = 1; 3677 cmd.data1 = mxge_rss_hash_type; 3678 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_RSS_ENABLE, &cmd); 3679 if (err != 0) { 3680 device_printf(sc->dev, "failed to enable slices\n"); 3681 return err; 3682 } 3683 } 3684 3685 3686 mxge_choose_params(sc->ifp->if_mtu, &big_bytes, &cl_size, &nbufs); 3687 3688 cmd.data0 = nbufs; 3689 err = mxge_send_cmd(sc, MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS, 3690 &cmd); 3691 /* error is only meaningful if we're trying to set 3692 MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS > 1 */ 3693 if (err && nbufs > 1) { 3694 device_printf(sc->dev, 3695 "Failed to set alway-use-n to %d\n", 3696 nbufs); 3697 return EIO; 3698 } 3699 /* Give the firmware the mtu and the big and small buffer 3700 sizes. The firmware wants the big buf size to be a power 3701 of two. Luckily, FreeBSD's clusters are powers of two */ 3702 cmd.data0 = sc->ifp->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3703 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_MTU, &cmd); 3704 cmd.data0 = MHLEN - MXGEFW_PAD; 3705 err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE, 3706 &cmd); 3707 cmd.data0 = big_bytes; 3708 err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd); 3709 3710 if (err != 0) { 3711 device_printf(sc->dev, "failed to setup params\n"); 3712 goto abort; 3713 } 3714 3715 /* Now give him the pointer to the stats block */ 3716 for (slice = 0; 3717 #ifdef IFNET_BUF_RING 3718 slice < sc->num_slices; 3719 #else 3720 slice < 1; 3721 #endif 3722 slice++) { 3723 ss = &sc->ss[slice]; 3724 cmd.data0 = 3725 MXGE_LOWPART_TO_U32(ss->fw_stats_dma.bus_addr); 3726 cmd.data1 = 3727 MXGE_HIGHPART_TO_U32(ss->fw_stats_dma.bus_addr); 3728 cmd.data2 = sizeof(struct mcp_irq_data); 3729 cmd.data2 |= (slice << 16); 3730 err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd); 3731 } 3732 3733 if (err != 0) { 3734 bus = sc->ss->fw_stats_dma.bus_addr; 3735 bus += offsetof(struct mcp_irq_data, send_done_count); 3736 cmd.data0 = MXGE_LOWPART_TO_U32(bus); 3737 cmd.data1 = MXGE_HIGHPART_TO_U32(bus); 3738 err = mxge_send_cmd(sc, 3739 MXGEFW_CMD_SET_STATS_DMA_OBSOLETE, 3740 &cmd); 3741 /* Firmware cannot support multicast without STATS_DMA_V2 */ 3742 sc->fw_multicast_support = 0; 3743 } else { 3744 sc->fw_multicast_support = 1; 3745 } 3746 3747 if (err != 0) { 3748 device_printf(sc->dev, "failed to setup params\n"); 3749 goto abort; 3750 } 3751 3752 for (slice = 0; slice < sc->num_slices; slice++) { 3753 err = mxge_slice_open(&sc->ss[slice], nbufs, cl_size); 3754 if (err != 0) { 3755 device_printf(sc->dev, "couldn't open slice %d\n", 3756 slice); 3757 goto abort; 3758 } 3759 } 3760 3761 /* Finally, start the firmware running */ 3762 err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_UP, &cmd); 3763 if (err) { 3764 device_printf(sc->dev, "Couldn't bring up link\n"); 3765 goto abort; 3766 } 3767 #ifdef IFNET_BUF_RING 3768 for (slice = 0; slice < sc->num_slices; slice++) { 3769 ss = &sc->ss[slice]; 3770 ss->if_drv_flags |= IFF_DRV_RUNNING; 3771 ss->if_drv_flags &= ~IFF_DRV_OACTIVE; 3772 } 3773 #endif 3774 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; 3775 sc->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 3776 3777 return 0; 3778 3779 3780 abort: 3781 mxge_free_mbufs(sc); 3782 3783 return err; 3784 } 3785 3786 static int 3787 mxge_close(mxge_softc_t *sc, int down) 3788 { 3789 mxge_cmd_t cmd; 3790 int err, old_down_cnt; 3791 #ifdef IFNET_BUF_RING 3792 struct mxge_slice_state *ss; 3793 int slice; 3794 #endif 3795 3796 #ifdef IFNET_BUF_RING 3797 for (slice = 0; slice < sc->num_slices; slice++) { 3798 ss = &sc->ss[slice]; 3799 ss->if_drv_flags &= ~IFF_DRV_RUNNING; 3800 } 3801 #endif 3802 sc->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3803 if (!down) { 3804 old_down_cnt = sc->down_cnt; 3805 wmb(); 3806 err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_DOWN, &cmd); 3807 if (err) { 3808 device_printf(sc->dev, 3809 "Couldn't bring down link\n"); 3810 } 3811 if (old_down_cnt == sc->down_cnt) { 3812 /* wait for down irq */ 3813 DELAY(10 * sc->intr_coal_delay); 3814 } 3815 wmb(); 3816 if (old_down_cnt == sc->down_cnt) { 3817 device_printf(sc->dev, "never got down irq\n"); 3818 } 3819 } 3820 mxge_free_mbufs(sc); 3821 3822 return 0; 3823 } 3824 3825 static void 3826 mxge_setup_cfg_space(mxge_softc_t *sc) 3827 { 3828 device_t dev = sc->dev; 3829 int reg; 3830 uint16_t cmd, lnk, pectl; 3831 3832 /* find the PCIe link width and set max read request to 4KB*/ 3833 if (pci_find_cap(dev, PCIY_EXPRESS, ®) == 0) { 3834 lnk = pci_read_config(dev, reg + 0x12, 2); 3835 sc->link_width = (lnk >> 4) & 0x3f; 3836 3837 if (sc->pectl == 0) { 3838 pectl = pci_read_config(dev, reg + 0x8, 2); 3839 pectl = (pectl & ~0x7000) | (5 << 12); 3840 pci_write_config(dev, reg + 0x8, pectl, 2); 3841 sc->pectl = pectl; 3842 } else { 3843 /* restore saved pectl after watchdog reset */ 3844 pci_write_config(dev, reg + 0x8, sc->pectl, 2); 3845 } 3846 } 3847 3848 /* Enable DMA and Memory space access */ 3849 pci_enable_busmaster(dev); 3850 cmd = pci_read_config(dev, PCIR_COMMAND, 2); 3851 cmd |= PCIM_CMD_MEMEN; 3852 pci_write_config(dev, PCIR_COMMAND, cmd, 2); 3853 } 3854 3855 static uint32_t 3856 mxge_read_reboot(mxge_softc_t *sc) 3857 { 3858 device_t dev = sc->dev; 3859 uint32_t vs; 3860 3861 /* find the vendor specific offset */ 3862 if (pci_find_cap(dev, PCIY_VENDOR, &vs) != 0) { 3863 device_printf(sc->dev, 3864 "could not find vendor specific offset\n"); 3865 return (uint32_t)-1; 3866 } 3867 /* enable read32 mode */ 3868 pci_write_config(dev, vs + 0x10, 0x3, 1); 3869 /* tell NIC which register to read */ 3870 pci_write_config(dev, vs + 0x18, 0xfffffff0, 4); 3871 return (pci_read_config(dev, vs + 0x14, 4)); 3872 } 3873 3874 static void 3875 mxge_watchdog_reset(mxge_softc_t *sc) 3876 { 3877 struct pci_devinfo *dinfo; 3878 struct mxge_slice_state *ss; 3879 int err, running, s, num_tx_slices = 1; 3880 uint32_t reboot; 3881 uint16_t cmd; 3882 3883 err = ENXIO; 3884 3885 device_printf(sc->dev, "Watchdog reset!\n"); 3886 3887 /* 3888 * check to see if the NIC rebooted. If it did, then all of 3889 * PCI config space has been reset, and things like the 3890 * busmaster bit will be zero. If this is the case, then we 3891 * must restore PCI config space before the NIC can be used 3892 * again 3893 */ 3894 cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2); 3895 if (cmd == 0xffff) { 3896 /* 3897 * maybe the watchdog caught the NIC rebooting; wait 3898 * up to 100ms for it to finish. If it does not come 3899 * back, then give up 3900 */ 3901 DELAY(1000*100); 3902 cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2); 3903 if (cmd == 0xffff) { 3904 device_printf(sc->dev, "NIC disappeared!\n"); 3905 } 3906 } 3907 if ((cmd & PCIM_CMD_BUSMASTEREN) == 0) { 3908 /* print the reboot status */ 3909 reboot = mxge_read_reboot(sc); 3910 device_printf(sc->dev, "NIC rebooted, status = 0x%x\n", 3911 reboot); 3912 running = sc->ifp->if_drv_flags & IFF_DRV_RUNNING; 3913 if (running) { 3914 3915 /* 3916 * quiesce NIC so that TX routines will not try to 3917 * xmit after restoration of BAR 3918 */ 3919 3920 /* Mark the link as down */ 3921 if (sc->link_state) { 3922 sc->link_state = 0; 3923 if_link_state_change(sc->ifp, 3924 LINK_STATE_DOWN); 3925 } 3926 #ifdef IFNET_BUF_RING 3927 num_tx_slices = sc->num_slices; 3928 #endif 3929 /* grab all TX locks to ensure no tx */ 3930 for (s = 0; s < num_tx_slices; s++) { 3931 ss = &sc->ss[s]; 3932 mtx_lock(&ss->tx.mtx); 3933 } 3934 mxge_close(sc, 1); 3935 } 3936 /* restore PCI configuration space */ 3937 dinfo = device_get_ivars(sc->dev); 3938 pci_cfg_restore(sc->dev, dinfo); 3939 3940 /* and redo any changes we made to our config space */ 3941 mxge_setup_cfg_space(sc); 3942 3943 /* reload f/w */ 3944 err = mxge_load_firmware(sc, 0); 3945 if (err) { 3946 device_printf(sc->dev, 3947 "Unable to re-load f/w\n"); 3948 } 3949 if (running) { 3950 if (!err) 3951 err = mxge_open(sc); 3952 /* release all TX locks */ 3953 for (s = 0; s < num_tx_slices; s++) { 3954 ss = &sc->ss[s]; 3955 #ifdef IFNET_BUF_RING 3956 mxge_start_locked(ss); 3957 #endif 3958 mtx_unlock(&ss->tx.mtx); 3959 } 3960 } 3961 sc->watchdog_resets++; 3962 } else { 3963 device_printf(sc->dev, 3964 "NIC did not reboot, not resetting\n"); 3965 err = 0; 3966 } 3967 if (err) { 3968 device_printf(sc->dev, "watchdog reset failed\n"); 3969 } else { 3970 if (sc->dying == 2) 3971 sc->dying = 0; 3972 callout_reset(&sc->co_hdl, mxge_ticks, mxge_tick, sc); 3973 } 3974 } 3975 3976 static void 3977 mxge_watchdog_task(void *arg, int pending) 3978 { 3979 mxge_softc_t *sc = arg; 3980 3981 3982 mtx_lock(&sc->driver_mtx); 3983 mxge_watchdog_reset(sc); 3984 mtx_unlock(&sc->driver_mtx); 3985 } 3986 3987 static void 3988 mxge_warn_stuck(mxge_softc_t *sc, mxge_tx_ring_t *tx, int slice) 3989 { 3990 tx = &sc->ss[slice].tx; 3991 device_printf(sc->dev, "slice %d struck? ring state:\n", slice); 3992 device_printf(sc->dev, 3993 "tx.req=%d tx.done=%d, tx.queue_active=%d\n", 3994 tx->req, tx->done, tx->queue_active); 3995 device_printf(sc->dev, "tx.activate=%d tx.deactivate=%d\n", 3996 tx->activate, tx->deactivate); 3997 device_printf(sc->dev, "pkt_done=%d fw=%d\n", 3998 tx->pkt_done, 3999 be32toh(sc->ss->fw_stats->send_done_count)); 4000 } 4001 4002 static int 4003 mxge_watchdog(mxge_softc_t *sc) 4004 { 4005 mxge_tx_ring_t *tx; 4006 uint32_t rx_pause = be32toh(sc->ss->fw_stats->dropped_pause); 4007 int i, err = 0; 4008 4009 /* see if we have outstanding transmits, which 4010 have been pending for more than mxge_ticks */ 4011 for (i = 0; 4012 #ifdef IFNET_BUF_RING 4013 (i < sc->num_slices) && (err == 0); 4014 #else 4015 (i < 1) && (err == 0); 4016 #endif 4017 i++) { 4018 tx = &sc->ss[i].tx; 4019 if (tx->req != tx->done && 4020 tx->watchdog_req != tx->watchdog_done && 4021 tx->done == tx->watchdog_done) { 4022 /* check for pause blocking before resetting */ 4023 if (tx->watchdog_rx_pause == rx_pause) { 4024 mxge_warn_stuck(sc, tx, i); 4025 taskqueue_enqueue(sc->tq, &sc->watchdog_task); 4026 return (ENXIO); 4027 } 4028 else 4029 device_printf(sc->dev, "Flow control blocking " 4030 "xmits, check link partner\n"); 4031 } 4032 4033 tx->watchdog_req = tx->req; 4034 tx->watchdog_done = tx->done; 4035 tx->watchdog_rx_pause = rx_pause; 4036 } 4037 4038 if (sc->need_media_probe) 4039 mxge_media_probe(sc); 4040 return (err); 4041 } 4042 4043 static u_long 4044 mxge_update_stats(mxge_softc_t *sc) 4045 { 4046 struct mxge_slice_state *ss; 4047 u_long pkts = 0; 4048 u_long ipackets = 0; 4049 u_long opackets = 0; 4050 #ifdef IFNET_BUF_RING 4051 u_long obytes = 0; 4052 u_long omcasts = 0; 4053 u_long odrops = 0; 4054 #endif 4055 u_long oerrors = 0; 4056 int slice; 4057 4058 for (slice = 0; slice < sc->num_slices; slice++) { 4059 ss = &sc->ss[slice]; 4060 ipackets += ss->ipackets; 4061 opackets += ss->opackets; 4062 #ifdef IFNET_BUF_RING 4063 obytes += ss->obytes; 4064 omcasts += ss->omcasts; 4065 odrops += ss->tx.br->br_drops; 4066 #endif 4067 oerrors += ss->oerrors; 4068 } 4069 pkts = (ipackets - sc->ifp->if_ipackets); 4070 pkts += (opackets - sc->ifp->if_opackets); 4071 sc->ifp->if_ipackets = ipackets; 4072 sc->ifp->if_opackets = opackets; 4073 #ifdef IFNET_BUF_RING 4074 sc->ifp->if_obytes = obytes; 4075 sc->ifp->if_omcasts = omcasts; 4076 sc->ifp->if_snd.ifq_drops = odrops; 4077 #endif 4078 sc->ifp->if_oerrors = oerrors; 4079 return pkts; 4080 } 4081 4082 static void 4083 mxge_tick(void *arg) 4084 { 4085 mxge_softc_t *sc = arg; 4086 u_long pkts = 0; 4087 int err = 0; 4088 int running, ticks; 4089 uint16_t cmd; 4090 4091 ticks = mxge_ticks; 4092 running = sc->ifp->if_drv_flags & IFF_DRV_RUNNING; 4093 if (running) { 4094 /* aggregate stats from different slices */ 4095 pkts = mxge_update_stats(sc); 4096 if (!sc->watchdog_countdown) { 4097 err = mxge_watchdog(sc); 4098 sc->watchdog_countdown = 4; 4099 } 4100 sc->watchdog_countdown--; 4101 } 4102 if (pkts == 0) { 4103 /* ensure NIC did not suffer h/w fault while idle */ 4104 cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2); 4105 if ((cmd & PCIM_CMD_BUSMASTEREN) == 0) { 4106 sc->dying = 2; 4107 taskqueue_enqueue(sc->tq, &sc->watchdog_task); 4108 err = ENXIO; 4109 } 4110 /* look less often if NIC is idle */ 4111 ticks *= 4; 4112 } 4113 4114 if (err == 0) 4115 callout_reset(&sc->co_hdl, ticks, mxge_tick, sc); 4116 4117 } 4118 4119 static int 4120 mxge_media_change(struct ifnet *ifp) 4121 { 4122 return EINVAL; 4123 } 4124 4125 static int 4126 mxge_change_mtu(mxge_softc_t *sc, int mtu) 4127 { 4128 struct ifnet *ifp = sc->ifp; 4129 int real_mtu, old_mtu; 4130 int err = 0; 4131 4132 4133 real_mtu = mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 4134 if ((real_mtu > sc->max_mtu) || real_mtu < 60) 4135 return EINVAL; 4136 mtx_lock(&sc->driver_mtx); 4137 old_mtu = ifp->if_mtu; 4138 ifp->if_mtu = mtu; 4139 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 4140 mxge_close(sc, 0); 4141 err = mxge_open(sc); 4142 if (err != 0) { 4143 ifp->if_mtu = old_mtu; 4144 mxge_close(sc, 0); 4145 (void) mxge_open(sc); 4146 } 4147 } 4148 mtx_unlock(&sc->driver_mtx); 4149 return err; 4150 } 4151 4152 static void 4153 mxge_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 4154 { 4155 mxge_softc_t *sc = ifp->if_softc; 4156 4157 4158 if (sc == NULL) 4159 return; 4160 ifmr->ifm_status = IFM_AVALID; 4161 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 4162 ifmr->ifm_status |= sc->link_state ? IFM_ACTIVE : 0; 4163 ifmr->ifm_active |= sc->current_media; 4164 } 4165 4166 static int 4167 mxge_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 4168 { 4169 mxge_softc_t *sc = ifp->if_softc; 4170 struct ifreq *ifr = (struct ifreq *)data; 4171 int err, mask; 4172 4173 err = 0; 4174 switch (command) { 4175 case SIOCSIFADDR: 4176 case SIOCGIFADDR: 4177 err = ether_ioctl(ifp, command, data); 4178 break; 4179 4180 case SIOCSIFMTU: 4181 err = mxge_change_mtu(sc, ifr->ifr_mtu); 4182 break; 4183 4184 case SIOCSIFFLAGS: 4185 mtx_lock(&sc->driver_mtx); 4186 if (sc->dying) { 4187 mtx_unlock(&sc->driver_mtx); 4188 return EINVAL; 4189 } 4190 if (ifp->if_flags & IFF_UP) { 4191 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4192 err = mxge_open(sc); 4193 } else { 4194 /* take care of promis can allmulti 4195 flag chages */ 4196 mxge_change_promisc(sc, 4197 ifp->if_flags & IFF_PROMISC); 4198 mxge_set_multicast_list(sc); 4199 } 4200 } else { 4201 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 4202 mxge_close(sc, 0); 4203 } 4204 } 4205 mtx_unlock(&sc->driver_mtx); 4206 break; 4207 4208 case SIOCADDMULTI: 4209 case SIOCDELMULTI: 4210 mtx_lock(&sc->driver_mtx); 4211 mxge_set_multicast_list(sc); 4212 mtx_unlock(&sc->driver_mtx); 4213 break; 4214 4215 case SIOCSIFCAP: 4216 mtx_lock(&sc->driver_mtx); 4217 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 4218 if (mask & IFCAP_TXCSUM) { 4219 if (IFCAP_TXCSUM & ifp->if_capenable) { 4220 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4); 4221 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP); 4222 } else { 4223 ifp->if_capenable |= IFCAP_TXCSUM; 4224 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 4225 } 4226 } else if (mask & IFCAP_RXCSUM) { 4227 if (IFCAP_RXCSUM & ifp->if_capenable) { 4228 ifp->if_capenable &= ~IFCAP_RXCSUM; 4229 } else { 4230 ifp->if_capenable |= IFCAP_RXCSUM; 4231 } 4232 } 4233 if (mask & IFCAP_TSO4) { 4234 if (IFCAP_TSO4 & ifp->if_capenable) { 4235 ifp->if_capenable &= ~IFCAP_TSO4; 4236 } else if (IFCAP_TXCSUM & ifp->if_capenable) { 4237 ifp->if_capenable |= IFCAP_TSO4; 4238 ifp->if_hwassist |= CSUM_TSO; 4239 } else { 4240 printf("mxge requires tx checksum offload" 4241 " be enabled to use TSO\n"); 4242 err = EINVAL; 4243 } 4244 } 4245 #if IFCAP_TSO6 4246 if (mask & IFCAP_TXCSUM_IPV6) { 4247 if (IFCAP_TXCSUM_IPV6 & ifp->if_capenable) { 4248 ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 4249 | IFCAP_TSO6); 4250 ifp->if_hwassist &= ~(CSUM_TCP_IPV6 4251 | CSUM_UDP); 4252 } else { 4253 ifp->if_capenable |= IFCAP_TXCSUM_IPV6; 4254 ifp->if_hwassist |= (CSUM_TCP_IPV6 4255 | CSUM_UDP_IPV6); 4256 } 4257 } else if (mask & IFCAP_RXCSUM_IPV6) { 4258 if (IFCAP_RXCSUM_IPV6 & ifp->if_capenable) { 4259 ifp->if_capenable &= ~IFCAP_RXCSUM_IPV6; 4260 } else { 4261 ifp->if_capenable |= IFCAP_RXCSUM_IPV6; 4262 } 4263 } 4264 if (mask & IFCAP_TSO6) { 4265 if (IFCAP_TSO6 & ifp->if_capenable) { 4266 ifp->if_capenable &= ~IFCAP_TSO6; 4267 } else if (IFCAP_TXCSUM_IPV6 & ifp->if_capenable) { 4268 ifp->if_capenable |= IFCAP_TSO6; 4269 ifp->if_hwassist |= CSUM_TSO; 4270 } else { 4271 printf("mxge requires tx checksum offload" 4272 " be enabled to use TSO\n"); 4273 err = EINVAL; 4274 } 4275 } 4276 #endif /*IFCAP_TSO6 */ 4277 4278 if (mask & IFCAP_LRO) 4279 ifp->if_capenable ^= IFCAP_LRO; 4280 if (mask & IFCAP_VLAN_HWTAGGING) 4281 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 4282 if (mask & IFCAP_VLAN_HWTSO) 4283 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 4284 4285 if (!(ifp->if_capabilities & IFCAP_VLAN_HWTSO) || 4286 !(ifp->if_capenable & IFCAP_VLAN_HWTAGGING)) 4287 ifp->if_capenable &= ~IFCAP_VLAN_HWTSO; 4288 4289 mtx_unlock(&sc->driver_mtx); 4290 VLAN_CAPABILITIES(ifp); 4291 4292 break; 4293 4294 case SIOCGIFMEDIA: 4295 mtx_lock(&sc->driver_mtx); 4296 mxge_media_probe(sc); 4297 mtx_unlock(&sc->driver_mtx); 4298 err = ifmedia_ioctl(ifp, (struct ifreq *)data, 4299 &sc->media, command); 4300 break; 4301 4302 default: 4303 err = ENOTTY; 4304 } 4305 return err; 4306 } 4307 4308 static void 4309 mxge_fetch_tunables(mxge_softc_t *sc) 4310 { 4311 4312 TUNABLE_INT_FETCH("hw.mxge.max_slices", &mxge_max_slices); 4313 TUNABLE_INT_FETCH("hw.mxge.flow_control_enabled", 4314 &mxge_flow_control); 4315 TUNABLE_INT_FETCH("hw.mxge.intr_coal_delay", 4316 &mxge_intr_coal_delay); 4317 TUNABLE_INT_FETCH("hw.mxge.nvidia_ecrc_enable", 4318 &mxge_nvidia_ecrc_enable); 4319 TUNABLE_INT_FETCH("hw.mxge.force_firmware", 4320 &mxge_force_firmware); 4321 TUNABLE_INT_FETCH("hw.mxge.deassert_wait", 4322 &mxge_deassert_wait); 4323 TUNABLE_INT_FETCH("hw.mxge.verbose", 4324 &mxge_verbose); 4325 TUNABLE_INT_FETCH("hw.mxge.ticks", &mxge_ticks); 4326 TUNABLE_INT_FETCH("hw.mxge.always_promisc", &mxge_always_promisc); 4327 TUNABLE_INT_FETCH("hw.mxge.rss_hash_type", &mxge_rss_hash_type); 4328 TUNABLE_INT_FETCH("hw.mxge.rss_hashtype", &mxge_rss_hash_type); 4329 TUNABLE_INT_FETCH("hw.mxge.initial_mtu", &mxge_initial_mtu); 4330 TUNABLE_INT_FETCH("hw.mxge.throttle", &mxge_throttle); 4331 4332 if (bootverbose) 4333 mxge_verbose = 1; 4334 if (mxge_intr_coal_delay < 0 || mxge_intr_coal_delay > 10*1000) 4335 mxge_intr_coal_delay = 30; 4336 if (mxge_ticks == 0) 4337 mxge_ticks = hz / 2; 4338 sc->pause = mxge_flow_control; 4339 if (mxge_rss_hash_type < MXGEFW_RSS_HASH_TYPE_IPV4 4340 || mxge_rss_hash_type > MXGEFW_RSS_HASH_TYPE_MAX) { 4341 mxge_rss_hash_type = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT; 4342 } 4343 if (mxge_initial_mtu > ETHERMTU_JUMBO || 4344 mxge_initial_mtu < ETHER_MIN_LEN) 4345 mxge_initial_mtu = ETHERMTU_JUMBO; 4346 4347 if (mxge_throttle && mxge_throttle > MXGE_MAX_THROTTLE) 4348 mxge_throttle = MXGE_MAX_THROTTLE; 4349 if (mxge_throttle && mxge_throttle < MXGE_MIN_THROTTLE) 4350 mxge_throttle = MXGE_MIN_THROTTLE; 4351 sc->throttle = mxge_throttle; 4352 } 4353 4354 4355 static void 4356 mxge_free_slices(mxge_softc_t *sc) 4357 { 4358 struct mxge_slice_state *ss; 4359 int i; 4360 4361 4362 if (sc->ss == NULL) 4363 return; 4364 4365 for (i = 0; i < sc->num_slices; i++) { 4366 ss = &sc->ss[i]; 4367 if (ss->fw_stats != NULL) { 4368 mxge_dma_free(&ss->fw_stats_dma); 4369 ss->fw_stats = NULL; 4370 #ifdef IFNET_BUF_RING 4371 if (ss->tx.br != NULL) { 4372 drbr_free(ss->tx.br, M_DEVBUF); 4373 ss->tx.br = NULL; 4374 } 4375 #endif 4376 mtx_destroy(&ss->tx.mtx); 4377 } 4378 if (ss->rx_done.entry != NULL) { 4379 mxge_dma_free(&ss->rx_done.dma); 4380 ss->rx_done.entry = NULL; 4381 } 4382 } 4383 free(sc->ss, M_DEVBUF); 4384 sc->ss = NULL; 4385 } 4386 4387 static int 4388 mxge_alloc_slices(mxge_softc_t *sc) 4389 { 4390 mxge_cmd_t cmd; 4391 struct mxge_slice_state *ss; 4392 size_t bytes; 4393 int err, i, max_intr_slots; 4394 4395 err = mxge_send_cmd(sc, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd); 4396 if (err != 0) { 4397 device_printf(sc->dev, "Cannot determine rx ring size\n"); 4398 return err; 4399 } 4400 sc->rx_ring_size = cmd.data0; 4401 max_intr_slots = 2 * (sc->rx_ring_size / sizeof (mcp_dma_addr_t)); 4402 4403 bytes = sizeof (*sc->ss) * sc->num_slices; 4404 sc->ss = malloc(bytes, M_DEVBUF, M_NOWAIT | M_ZERO); 4405 if (sc->ss == NULL) 4406 return (ENOMEM); 4407 for (i = 0; i < sc->num_slices; i++) { 4408 ss = &sc->ss[i]; 4409 4410 ss->sc = sc; 4411 4412 /* allocate per-slice rx interrupt queues */ 4413 4414 bytes = max_intr_slots * sizeof (*ss->rx_done.entry); 4415 err = mxge_dma_alloc(sc, &ss->rx_done.dma, bytes, 4096); 4416 if (err != 0) 4417 goto abort; 4418 ss->rx_done.entry = ss->rx_done.dma.addr; 4419 bzero(ss->rx_done.entry, bytes); 4420 4421 /* 4422 * allocate the per-slice firmware stats; stats 4423 * (including tx) are used used only on the first 4424 * slice for now 4425 */ 4426 #ifndef IFNET_BUF_RING 4427 if (i > 0) 4428 continue; 4429 #endif 4430 4431 bytes = sizeof (*ss->fw_stats); 4432 err = mxge_dma_alloc(sc, &ss->fw_stats_dma, 4433 sizeof (*ss->fw_stats), 64); 4434 if (err != 0) 4435 goto abort; 4436 ss->fw_stats = (mcp_irq_data_t *)ss->fw_stats_dma.addr; 4437 snprintf(ss->tx.mtx_name, sizeof(ss->tx.mtx_name), 4438 "%s:tx(%d)", device_get_nameunit(sc->dev), i); 4439 mtx_init(&ss->tx.mtx, ss->tx.mtx_name, NULL, MTX_DEF); 4440 #ifdef IFNET_BUF_RING 4441 ss->tx.br = buf_ring_alloc(2048, M_DEVBUF, M_WAITOK, 4442 &ss->tx.mtx); 4443 #endif 4444 } 4445 4446 return (0); 4447 4448 abort: 4449 mxge_free_slices(sc); 4450 return (ENOMEM); 4451 } 4452 4453 static void 4454 mxge_slice_probe(mxge_softc_t *sc) 4455 { 4456 mxge_cmd_t cmd; 4457 char *old_fw; 4458 int msix_cnt, status, max_intr_slots; 4459 4460 sc->num_slices = 1; 4461 /* 4462 * don't enable multiple slices if they are not enabled, 4463 * or if this is not an SMP system 4464 */ 4465 4466 if (mxge_max_slices == 0 || mxge_max_slices == 1 || mp_ncpus < 2) 4467 return; 4468 4469 /* see how many MSI-X interrupts are available */ 4470 msix_cnt = pci_msix_count(sc->dev); 4471 if (msix_cnt < 2) 4472 return; 4473 4474 /* now load the slice aware firmware see what it supports */ 4475 old_fw = sc->fw_name; 4476 if (old_fw == mxge_fw_aligned) 4477 sc->fw_name = mxge_fw_rss_aligned; 4478 else 4479 sc->fw_name = mxge_fw_rss_unaligned; 4480 status = mxge_load_firmware(sc, 0); 4481 if (status != 0) { 4482 device_printf(sc->dev, "Falling back to a single slice\n"); 4483 return; 4484 } 4485 4486 /* try to send a reset command to the card to see if it 4487 is alive */ 4488 memset(&cmd, 0, sizeof (cmd)); 4489 status = mxge_send_cmd(sc, MXGEFW_CMD_RESET, &cmd); 4490 if (status != 0) { 4491 device_printf(sc->dev, "failed reset\n"); 4492 goto abort_with_fw; 4493 } 4494 4495 /* get rx ring size */ 4496 status = mxge_send_cmd(sc, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd); 4497 if (status != 0) { 4498 device_printf(sc->dev, "Cannot determine rx ring size\n"); 4499 goto abort_with_fw; 4500 } 4501 max_intr_slots = 2 * (cmd.data0 / sizeof (mcp_dma_addr_t)); 4502 4503 /* tell it the size of the interrupt queues */ 4504 cmd.data0 = max_intr_slots * sizeof (struct mcp_slot); 4505 status = mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd); 4506 if (status != 0) { 4507 device_printf(sc->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n"); 4508 goto abort_with_fw; 4509 } 4510 4511 /* ask the maximum number of slices it supports */ 4512 status = mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd); 4513 if (status != 0) { 4514 device_printf(sc->dev, 4515 "failed MXGEFW_CMD_GET_MAX_RSS_QUEUES\n"); 4516 goto abort_with_fw; 4517 } 4518 sc->num_slices = cmd.data0; 4519 if (sc->num_slices > msix_cnt) 4520 sc->num_slices = msix_cnt; 4521 4522 if (mxge_max_slices == -1) { 4523 /* cap to number of CPUs in system */ 4524 if (sc->num_slices > mp_ncpus) 4525 sc->num_slices = mp_ncpus; 4526 } else { 4527 if (sc->num_slices > mxge_max_slices) 4528 sc->num_slices = mxge_max_slices; 4529 } 4530 /* make sure it is a power of two */ 4531 while (sc->num_slices & (sc->num_slices - 1)) 4532 sc->num_slices--; 4533 4534 if (mxge_verbose) 4535 device_printf(sc->dev, "using %d slices\n", 4536 sc->num_slices); 4537 4538 return; 4539 4540 abort_with_fw: 4541 sc->fw_name = old_fw; 4542 (void) mxge_load_firmware(sc, 0); 4543 } 4544 4545 static int 4546 mxge_add_msix_irqs(mxge_softc_t *sc) 4547 { 4548 size_t bytes; 4549 int count, err, i, rid; 4550 4551 rid = PCIR_BAR(2); 4552 sc->msix_table_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 4553 &rid, RF_ACTIVE); 4554 4555 if (sc->msix_table_res == NULL) { 4556 device_printf(sc->dev, "couldn't alloc MSIX table res\n"); 4557 return ENXIO; 4558 } 4559 4560 count = sc->num_slices; 4561 err = pci_alloc_msix(sc->dev, &count); 4562 if (err != 0) { 4563 device_printf(sc->dev, "pci_alloc_msix: failed, wanted %d" 4564 "err = %d \n", sc->num_slices, err); 4565 goto abort_with_msix_table; 4566 } 4567 if (count < sc->num_slices) { 4568 device_printf(sc->dev, "pci_alloc_msix: need %d, got %d\n", 4569 count, sc->num_slices); 4570 device_printf(sc->dev, 4571 "Try setting hw.mxge.max_slices to %d\n", 4572 count); 4573 err = ENOSPC; 4574 goto abort_with_msix; 4575 } 4576 bytes = sizeof (*sc->msix_irq_res) * sc->num_slices; 4577 sc->msix_irq_res = malloc(bytes, M_DEVBUF, M_NOWAIT|M_ZERO); 4578 if (sc->msix_irq_res == NULL) { 4579 err = ENOMEM; 4580 goto abort_with_msix; 4581 } 4582 4583 for (i = 0; i < sc->num_slices; i++) { 4584 rid = i + 1; 4585 sc->msix_irq_res[i] = bus_alloc_resource_any(sc->dev, 4586 SYS_RES_IRQ, 4587 &rid, RF_ACTIVE); 4588 if (sc->msix_irq_res[i] == NULL) { 4589 device_printf(sc->dev, "couldn't allocate IRQ res" 4590 " for message %d\n", i); 4591 err = ENXIO; 4592 goto abort_with_res; 4593 } 4594 } 4595 4596 bytes = sizeof (*sc->msix_ih) * sc->num_slices; 4597 sc->msix_ih = malloc(bytes, M_DEVBUF, M_NOWAIT|M_ZERO); 4598 4599 for (i = 0; i < sc->num_slices; i++) { 4600 err = bus_setup_intr(sc->dev, sc->msix_irq_res[i], 4601 INTR_TYPE_NET | INTR_MPSAFE, 4602 #if __FreeBSD_version > 700030 4603 NULL, 4604 #endif 4605 mxge_intr, &sc->ss[i], &sc->msix_ih[i]); 4606 if (err != 0) { 4607 device_printf(sc->dev, "couldn't setup intr for " 4608 "message %d\n", i); 4609 goto abort_with_intr; 4610 } 4611 bus_describe_intr(sc->dev, sc->msix_irq_res[i], 4612 sc->msix_ih[i], "s%d", i); 4613 } 4614 4615 if (mxge_verbose) { 4616 device_printf(sc->dev, "using %d msix IRQs:", 4617 sc->num_slices); 4618 for (i = 0; i < sc->num_slices; i++) 4619 printf(" %ld", rman_get_start(sc->msix_irq_res[i])); 4620 printf("\n"); 4621 } 4622 return (0); 4623 4624 abort_with_intr: 4625 for (i = 0; i < sc->num_slices; i++) { 4626 if (sc->msix_ih[i] != NULL) { 4627 bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 4628 sc->msix_ih[i]); 4629 sc->msix_ih[i] = NULL; 4630 } 4631 } 4632 free(sc->msix_ih, M_DEVBUF); 4633 4634 4635 abort_with_res: 4636 for (i = 0; i < sc->num_slices; i++) { 4637 rid = i + 1; 4638 if (sc->msix_irq_res[i] != NULL) 4639 bus_release_resource(sc->dev, SYS_RES_IRQ, rid, 4640 sc->msix_irq_res[i]); 4641 sc->msix_irq_res[i] = NULL; 4642 } 4643 free(sc->msix_irq_res, M_DEVBUF); 4644 4645 4646 abort_with_msix: 4647 pci_release_msi(sc->dev); 4648 4649 abort_with_msix_table: 4650 bus_release_resource(sc->dev, SYS_RES_MEMORY, PCIR_BAR(2), 4651 sc->msix_table_res); 4652 4653 return err; 4654 } 4655 4656 static int 4657 mxge_add_single_irq(mxge_softc_t *sc) 4658 { 4659 int count, err, rid; 4660 4661 count = pci_msi_count(sc->dev); 4662 if (count == 1 && pci_alloc_msi(sc->dev, &count) == 0) { 4663 rid = 1; 4664 } else { 4665 rid = 0; 4666 sc->legacy_irq = 1; 4667 } 4668 sc->irq_res = bus_alloc_resource(sc->dev, SYS_RES_IRQ, &rid, 0, ~0, 4669 1, RF_SHAREABLE | RF_ACTIVE); 4670 if (sc->irq_res == NULL) { 4671 device_printf(sc->dev, "could not alloc interrupt\n"); 4672 return ENXIO; 4673 } 4674 if (mxge_verbose) 4675 device_printf(sc->dev, "using %s irq %ld\n", 4676 sc->legacy_irq ? "INTx" : "MSI", 4677 rman_get_start(sc->irq_res)); 4678 err = bus_setup_intr(sc->dev, sc->irq_res, 4679 INTR_TYPE_NET | INTR_MPSAFE, 4680 #if __FreeBSD_version > 700030 4681 NULL, 4682 #endif 4683 mxge_intr, &sc->ss[0], &sc->ih); 4684 if (err != 0) { 4685 bus_release_resource(sc->dev, SYS_RES_IRQ, 4686 sc->legacy_irq ? 0 : 1, sc->irq_res); 4687 if (!sc->legacy_irq) 4688 pci_release_msi(sc->dev); 4689 } 4690 return err; 4691 } 4692 4693 static void 4694 mxge_rem_msix_irqs(mxge_softc_t *sc) 4695 { 4696 int i, rid; 4697 4698 for (i = 0; i < sc->num_slices; i++) { 4699 if (sc->msix_ih[i] != NULL) { 4700 bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 4701 sc->msix_ih[i]); 4702 sc->msix_ih[i] = NULL; 4703 } 4704 } 4705 free(sc->msix_ih, M_DEVBUF); 4706 4707 for (i = 0; i < sc->num_slices; i++) { 4708 rid = i + 1; 4709 if (sc->msix_irq_res[i] != NULL) 4710 bus_release_resource(sc->dev, SYS_RES_IRQ, rid, 4711 sc->msix_irq_res[i]); 4712 sc->msix_irq_res[i] = NULL; 4713 } 4714 free(sc->msix_irq_res, M_DEVBUF); 4715 4716 bus_release_resource(sc->dev, SYS_RES_MEMORY, PCIR_BAR(2), 4717 sc->msix_table_res); 4718 4719 pci_release_msi(sc->dev); 4720 return; 4721 } 4722 4723 static void 4724 mxge_rem_single_irq(mxge_softc_t *sc) 4725 { 4726 bus_teardown_intr(sc->dev, sc->irq_res, sc->ih); 4727 bus_release_resource(sc->dev, SYS_RES_IRQ, 4728 sc->legacy_irq ? 0 : 1, sc->irq_res); 4729 if (!sc->legacy_irq) 4730 pci_release_msi(sc->dev); 4731 } 4732 4733 static void 4734 mxge_rem_irq(mxge_softc_t *sc) 4735 { 4736 if (sc->num_slices > 1) 4737 mxge_rem_msix_irqs(sc); 4738 else 4739 mxge_rem_single_irq(sc); 4740 } 4741 4742 static int 4743 mxge_add_irq(mxge_softc_t *sc) 4744 { 4745 int err; 4746 4747 if (sc->num_slices > 1) 4748 err = mxge_add_msix_irqs(sc); 4749 else 4750 err = mxge_add_single_irq(sc); 4751 4752 if (0 && err == 0 && sc->num_slices > 1) { 4753 mxge_rem_msix_irqs(sc); 4754 err = mxge_add_msix_irqs(sc); 4755 } 4756 return err; 4757 } 4758 4759 4760 static int 4761 mxge_attach(device_t dev) 4762 { 4763 mxge_cmd_t cmd; 4764 mxge_softc_t *sc = device_get_softc(dev); 4765 struct ifnet *ifp; 4766 int err, rid; 4767 4768 sc->dev = dev; 4769 mxge_fetch_tunables(sc); 4770 4771 TASK_INIT(&sc->watchdog_task, 1, mxge_watchdog_task, sc); 4772 sc->tq = taskqueue_create("mxge_taskq", M_WAITOK, 4773 taskqueue_thread_enqueue, &sc->tq); 4774 if (sc->tq == NULL) { 4775 err = ENOMEM; 4776 goto abort_with_nothing; 4777 } 4778 4779 err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 4780 1, /* alignment */ 4781 0, /* boundary */ 4782 BUS_SPACE_MAXADDR, /* low */ 4783 BUS_SPACE_MAXADDR, /* high */ 4784 NULL, NULL, /* filter */ 4785 65536 + 256, /* maxsize */ 4786 MXGE_MAX_SEND_DESC, /* num segs */ 4787 65536, /* maxsegsize */ 4788 0, /* flags */ 4789 NULL, NULL, /* lock */ 4790 &sc->parent_dmat); /* tag */ 4791 4792 if (err != 0) { 4793 device_printf(sc->dev, "Err %d allocating parent dmat\n", 4794 err); 4795 goto abort_with_tq; 4796 } 4797 4798 ifp = sc->ifp = if_alloc(IFT_ETHER); 4799 if (ifp == NULL) { 4800 device_printf(dev, "can not if_alloc()\n"); 4801 err = ENOSPC; 4802 goto abort_with_parent_dmat; 4803 } 4804 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 4805 4806 snprintf(sc->cmd_mtx_name, sizeof(sc->cmd_mtx_name), "%s:cmd", 4807 device_get_nameunit(dev)); 4808 mtx_init(&sc->cmd_mtx, sc->cmd_mtx_name, NULL, MTX_DEF); 4809 snprintf(sc->driver_mtx_name, sizeof(sc->driver_mtx_name), 4810 "%s:drv", device_get_nameunit(dev)); 4811 mtx_init(&sc->driver_mtx, sc->driver_mtx_name, 4812 MTX_NETWORK_LOCK, MTX_DEF); 4813 4814 callout_init_mtx(&sc->co_hdl, &sc->driver_mtx, 0); 4815 4816 mxge_setup_cfg_space(sc); 4817 4818 /* Map the board into the kernel */ 4819 rid = PCIR_BARS; 4820 sc->mem_res = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid, 0, 4821 ~0, 1, RF_ACTIVE); 4822 if (sc->mem_res == NULL) { 4823 device_printf(dev, "could not map memory\n"); 4824 err = ENXIO; 4825 goto abort_with_lock; 4826 } 4827 sc->sram = rman_get_virtual(sc->mem_res); 4828 sc->sram_size = 2*1024*1024 - (2*(48*1024)+(32*1024)) - 0x100; 4829 if (sc->sram_size > rman_get_size(sc->mem_res)) { 4830 device_printf(dev, "impossible memory region size %ld\n", 4831 rman_get_size(sc->mem_res)); 4832 err = ENXIO; 4833 goto abort_with_mem_res; 4834 } 4835 4836 /* make NULL terminated copy of the EEPROM strings section of 4837 lanai SRAM */ 4838 bzero(sc->eeprom_strings, MXGE_EEPROM_STRINGS_SIZE); 4839 bus_space_read_region_1(rman_get_bustag(sc->mem_res), 4840 rman_get_bushandle(sc->mem_res), 4841 sc->sram_size - MXGE_EEPROM_STRINGS_SIZE, 4842 sc->eeprom_strings, 4843 MXGE_EEPROM_STRINGS_SIZE - 2); 4844 err = mxge_parse_strings(sc); 4845 if (err != 0) 4846 goto abort_with_mem_res; 4847 4848 /* Enable write combining for efficient use of PCIe bus */ 4849 mxge_enable_wc(sc); 4850 4851 /* Allocate the out of band dma memory */ 4852 err = mxge_dma_alloc(sc, &sc->cmd_dma, 4853 sizeof (mxge_cmd_t), 64); 4854 if (err != 0) 4855 goto abort_with_mem_res; 4856 sc->cmd = (mcp_cmd_response_t *) sc->cmd_dma.addr; 4857 err = mxge_dma_alloc(sc, &sc->zeropad_dma, 64, 64); 4858 if (err != 0) 4859 goto abort_with_cmd_dma; 4860 4861 err = mxge_dma_alloc(sc, &sc->dmabench_dma, 4096, 4096); 4862 if (err != 0) 4863 goto abort_with_zeropad_dma; 4864 4865 /* select & load the firmware */ 4866 err = mxge_select_firmware(sc); 4867 if (err != 0) 4868 goto abort_with_dmabench; 4869 sc->intr_coal_delay = mxge_intr_coal_delay; 4870 4871 mxge_slice_probe(sc); 4872 err = mxge_alloc_slices(sc); 4873 if (err != 0) 4874 goto abort_with_dmabench; 4875 4876 err = mxge_reset(sc, 0); 4877 if (err != 0) 4878 goto abort_with_slices; 4879 4880 err = mxge_alloc_rings(sc); 4881 if (err != 0) { 4882 device_printf(sc->dev, "failed to allocate rings\n"); 4883 goto abort_with_slices; 4884 } 4885 4886 err = mxge_add_irq(sc); 4887 if (err != 0) { 4888 device_printf(sc->dev, "failed to add irq\n"); 4889 goto abort_with_rings; 4890 } 4891 4892 if_initbaudrate(ifp, IF_Gbps(10)); 4893 ifp->if_capabilities = IFCAP_RXCSUM | IFCAP_TXCSUM | IFCAP_TSO4 | 4894 IFCAP_VLAN_MTU | IFCAP_LINKSTATE | IFCAP_TXCSUM_IPV6 | 4895 IFCAP_RXCSUM_IPV6; 4896 #if defined(INET) || defined(INET6) 4897 ifp->if_capabilities |= IFCAP_LRO; 4898 #endif 4899 4900 #ifdef MXGE_NEW_VLAN_API 4901 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM; 4902 4903 /* Only FW 1.4.32 and newer can do TSO over vlans */ 4904 if (sc->fw_ver_major == 1 && sc->fw_ver_minor == 4 && 4905 sc->fw_ver_tiny >= 32) 4906 ifp->if_capabilities |= IFCAP_VLAN_HWTSO; 4907 #endif 4908 sc->max_mtu = mxge_max_mtu(sc); 4909 if (sc->max_mtu >= 9000) 4910 ifp->if_capabilities |= IFCAP_JUMBO_MTU; 4911 else 4912 device_printf(dev, "MTU limited to %d. Install " 4913 "latest firmware for 9000 byte jumbo support\n", 4914 sc->max_mtu - ETHER_HDR_LEN); 4915 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_TSO; 4916 ifp->if_hwassist |= CSUM_TCP_IPV6 | CSUM_UDP_IPV6; 4917 /* check to see if f/w supports TSO for IPv6 */ 4918 if (!mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_TSO6_HDR_SIZE, &cmd)) { 4919 if (CSUM_TCP_IPV6) 4920 ifp->if_capabilities |= IFCAP_TSO6; 4921 sc->max_tso6_hlen = min(cmd.data0, 4922 sizeof (sc->ss[0].scratch)); 4923 } 4924 ifp->if_capenable = ifp->if_capabilities; 4925 if (sc->lro_cnt == 0) 4926 ifp->if_capenable &= ~IFCAP_LRO; 4927 ifp->if_init = mxge_init; 4928 ifp->if_softc = sc; 4929 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 4930 ifp->if_ioctl = mxge_ioctl; 4931 ifp->if_start = mxge_start; 4932 /* Initialise the ifmedia structure */ 4933 ifmedia_init(&sc->media, 0, mxge_media_change, 4934 mxge_media_status); 4935 mxge_media_init(sc); 4936 mxge_media_probe(sc); 4937 sc->dying = 0; 4938 ether_ifattach(ifp, sc->mac_addr); 4939 /* ether_ifattach sets mtu to ETHERMTU */ 4940 if (mxge_initial_mtu != ETHERMTU) 4941 mxge_change_mtu(sc, mxge_initial_mtu); 4942 4943 mxge_add_sysctls(sc); 4944 #ifdef IFNET_BUF_RING 4945 ifp->if_transmit = mxge_transmit; 4946 ifp->if_qflush = mxge_qflush; 4947 #endif 4948 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq", 4949 device_get_nameunit(sc->dev)); 4950 callout_reset(&sc->co_hdl, mxge_ticks, mxge_tick, sc); 4951 return 0; 4952 4953 abort_with_rings: 4954 mxge_free_rings(sc); 4955 abort_with_slices: 4956 mxge_free_slices(sc); 4957 abort_with_dmabench: 4958 mxge_dma_free(&sc->dmabench_dma); 4959 abort_with_zeropad_dma: 4960 mxge_dma_free(&sc->zeropad_dma); 4961 abort_with_cmd_dma: 4962 mxge_dma_free(&sc->cmd_dma); 4963 abort_with_mem_res: 4964 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BARS, sc->mem_res); 4965 abort_with_lock: 4966 pci_disable_busmaster(dev); 4967 mtx_destroy(&sc->cmd_mtx); 4968 mtx_destroy(&sc->driver_mtx); 4969 if_free(ifp); 4970 abort_with_parent_dmat: 4971 bus_dma_tag_destroy(sc->parent_dmat); 4972 abort_with_tq: 4973 if (sc->tq != NULL) { 4974 taskqueue_drain(sc->tq, &sc->watchdog_task); 4975 taskqueue_free(sc->tq); 4976 sc->tq = NULL; 4977 } 4978 abort_with_nothing: 4979 return err; 4980 } 4981 4982 static int 4983 mxge_detach(device_t dev) 4984 { 4985 mxge_softc_t *sc = device_get_softc(dev); 4986 4987 if (mxge_vlans_active(sc)) { 4988 device_printf(sc->dev, 4989 "Detach vlans before removing module\n"); 4990 return EBUSY; 4991 } 4992 mtx_lock(&sc->driver_mtx); 4993 sc->dying = 1; 4994 if (sc->ifp->if_drv_flags & IFF_DRV_RUNNING) 4995 mxge_close(sc, 0); 4996 mtx_unlock(&sc->driver_mtx); 4997 ether_ifdetach(sc->ifp); 4998 if (sc->tq != NULL) { 4999 taskqueue_drain(sc->tq, &sc->watchdog_task); 5000 taskqueue_free(sc->tq); 5001 sc->tq = NULL; 5002 } 5003 callout_drain(&sc->co_hdl); 5004 ifmedia_removeall(&sc->media); 5005 mxge_dummy_rdma(sc, 0); 5006 mxge_rem_sysctls(sc); 5007 mxge_rem_irq(sc); 5008 mxge_free_rings(sc); 5009 mxge_free_slices(sc); 5010 mxge_dma_free(&sc->dmabench_dma); 5011 mxge_dma_free(&sc->zeropad_dma); 5012 mxge_dma_free(&sc->cmd_dma); 5013 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BARS, sc->mem_res); 5014 pci_disable_busmaster(dev); 5015 mtx_destroy(&sc->cmd_mtx); 5016 mtx_destroy(&sc->driver_mtx); 5017 if_free(sc->ifp); 5018 bus_dma_tag_destroy(sc->parent_dmat); 5019 return 0; 5020 } 5021 5022 static int 5023 mxge_shutdown(device_t dev) 5024 { 5025 return 0; 5026 } 5027 5028 /* 5029 This file uses Myri10GE driver indentation. 5030 5031 Local Variables: 5032 c-file-style:"linux" 5033 tab-width:8 5034 End: 5035 */ 5036