16d87a65dSAndrew Gallatin /****************************************************************************** 2718cf2ccSPedro F. Giffuni SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3b2fc195eSAndrew Gallatin 4cabc512fSAndrew Gallatin Copyright (c) 2006-2013, Myricom Inc. 5b2fc195eSAndrew Gallatin All rights reserved. 6b2fc195eSAndrew Gallatin 7b2fc195eSAndrew Gallatin Redistribution and use in source and binary forms, with or without 8b2fc195eSAndrew Gallatin modification, are permitted provided that the following conditions are met: 9b2fc195eSAndrew Gallatin 10b2fc195eSAndrew Gallatin 1. Redistributions of source code must retain the above copyright notice, 11b2fc195eSAndrew Gallatin this list of conditions and the following disclaimer. 12b2fc195eSAndrew Gallatin 13eb8e82f5SAndrew Gallatin 2. Neither the name of the Myricom Inc, nor the names of its 14b2fc195eSAndrew Gallatin contributors may be used to endorse or promote products derived from 15b2fc195eSAndrew Gallatin this software without specific prior written permission. 16b2fc195eSAndrew Gallatin 17b2fc195eSAndrew Gallatin THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18b2fc195eSAndrew Gallatin AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19b2fc195eSAndrew Gallatin IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20b2fc195eSAndrew Gallatin ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21b2fc195eSAndrew Gallatin LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22b2fc195eSAndrew Gallatin CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23b2fc195eSAndrew Gallatin SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24b2fc195eSAndrew Gallatin INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25b2fc195eSAndrew Gallatin CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26b2fc195eSAndrew Gallatin ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27b2fc195eSAndrew Gallatin POSSIBILITY OF SUCH DAMAGE. 28b2fc195eSAndrew Gallatin 29b2fc195eSAndrew Gallatin ***************************************************************************/ 30b2fc195eSAndrew Gallatin 31b2fc195eSAndrew Gallatin #include <sys/cdefs.h> 32b2fc195eSAndrew Gallatin __FBSDID("$FreeBSD$"); 33b2fc195eSAndrew Gallatin 34b2fc195eSAndrew Gallatin #include <sys/param.h> 35b2fc195eSAndrew Gallatin #include <sys/systm.h> 36b2fc195eSAndrew Gallatin #include <sys/linker.h> 37b2fc195eSAndrew Gallatin #include <sys/firmware.h> 38b2fc195eSAndrew Gallatin #include <sys/endian.h> 39b2fc195eSAndrew Gallatin #include <sys/sockio.h> 40b2fc195eSAndrew Gallatin #include <sys/mbuf.h> 41b2fc195eSAndrew Gallatin #include <sys/malloc.h> 42b2fc195eSAndrew Gallatin #include <sys/kdb.h> 43b2fc195eSAndrew Gallatin #include <sys/kernel.h> 444e7f640dSJohn Baldwin #include <sys/lock.h> 45b2fc195eSAndrew Gallatin #include <sys/module.h> 46b2fc195eSAndrew Gallatin #include <sys/socket.h> 47b2fc195eSAndrew Gallatin #include <sys/sysctl.h> 48b2fc195eSAndrew Gallatin #include <sys/sx.h> 4972c042dfSAndrew Gallatin #include <sys/taskqueue.h> 50d9db5225SCraig Rodrigues #include <sys/zlib.h> 51b2fc195eSAndrew Gallatin 52b2fc195eSAndrew Gallatin #include <net/if.h> 5376039bc8SGleb Smirnoff #include <net/if_var.h> 54b2fc195eSAndrew Gallatin #include <net/if_arp.h> 55b2fc195eSAndrew Gallatin #include <net/ethernet.h> 56b2fc195eSAndrew Gallatin #include <net/if_dl.h> 57b2fc195eSAndrew Gallatin #include <net/if_media.h> 58b2fc195eSAndrew Gallatin 59b2fc195eSAndrew Gallatin #include <net/bpf.h> 60b2fc195eSAndrew Gallatin 61b2fc195eSAndrew Gallatin #include <net/if_types.h> 62b2fc195eSAndrew Gallatin #include <net/if_vlan_var.h> 63b2fc195eSAndrew Gallatin 64b2fc195eSAndrew Gallatin #include <netinet/in_systm.h> 65b2fc195eSAndrew Gallatin #include <netinet/in.h> 66b2fc195eSAndrew Gallatin #include <netinet/ip.h> 670a7a780eSAndrew Gallatin #include <netinet/ip6.h> 68aed8e389SAndrew Gallatin #include <netinet/tcp.h> 6926dd49c6SAndrew Gallatin #include <netinet/tcp_lro.h> 700a7a780eSAndrew Gallatin #include <netinet6/ip6_var.h> 71b2fc195eSAndrew Gallatin 72b2fc195eSAndrew Gallatin #include <machine/bus.h> 73053e637fSAndrew Gallatin #include <machine/in_cksum.h> 74b2fc195eSAndrew Gallatin #include <machine/resource.h> 75b2fc195eSAndrew Gallatin #include <sys/bus.h> 76b2fc195eSAndrew Gallatin #include <sys/rman.h> 771e413cf9SAndrew Gallatin #include <sys/smp.h> 78b2fc195eSAndrew Gallatin 79b2fc195eSAndrew Gallatin #include <dev/pci/pcireg.h> 80b2fc195eSAndrew Gallatin #include <dev/pci/pcivar.h> 81e749ef6bSAndrew Gallatin #include <dev/pci/pci_private.h> /* XXX for pci_cfg_restore */ 82b2fc195eSAndrew Gallatin 83b2fc195eSAndrew Gallatin #include <vm/vm.h> /* for pmap_mapdev() */ 84b2fc195eSAndrew Gallatin #include <vm/pmap.h> 85b2fc195eSAndrew Gallatin 86c2c14a69SAndrew Gallatin #if defined(__i386) || defined(__amd64) 87c2c14a69SAndrew Gallatin #include <machine/specialreg.h> 88c2c14a69SAndrew Gallatin #endif 89c2c14a69SAndrew Gallatin 906d87a65dSAndrew Gallatin #include <dev/mxge/mxge_mcp.h> 916d87a65dSAndrew Gallatin #include <dev/mxge/mcp_gen_header.h> 921e413cf9SAndrew Gallatin /*#define MXGE_FAKE_IFP*/ 936d87a65dSAndrew Gallatin #include <dev/mxge/if_mxge_var.h> 94869c7348SAndrew Gallatin #ifdef IFNET_BUF_RING 95869c7348SAndrew Gallatin #include <sys/buf_ring.h> 96869c7348SAndrew Gallatin #endif 97b2fc195eSAndrew Gallatin 98eb6219e3SAndrew Gallatin #include "opt_inet.h" 990a7a780eSAndrew Gallatin #include "opt_inet6.h" 100eb6219e3SAndrew Gallatin 101b2fc195eSAndrew Gallatin /* tunable params */ 1026d87a65dSAndrew Gallatin static int mxge_nvidia_ecrc_enable = 1; 103d91b1b49SAndrew Gallatin static int mxge_force_firmware = 0; 1046d87a65dSAndrew Gallatin static int mxge_intr_coal_delay = 30; 1055e7d8541SAndrew Gallatin static int mxge_deassert_wait = 1; 1066d87a65dSAndrew Gallatin static int mxge_flow_control = 1; 1075e7d8541SAndrew Gallatin static int mxge_verbose = 0; 108dce01b9bSAndrew Gallatin static int mxge_ticks; 1091e413cf9SAndrew Gallatin static int mxge_max_slices = 1; 1105769c5efSAndrew Gallatin static int mxge_rss_hash_type = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT; 1111e413cf9SAndrew Gallatin static int mxge_always_promisc = 0; 112f9453025SAndrew Gallatin static int mxge_initial_mtu = ETHERMTU_JUMBO; 11365c69066SAndrew Gallatin static int mxge_throttle = 0; 1146d87a65dSAndrew Gallatin static char *mxge_fw_unaligned = "mxge_ethp_z8e"; 1156d87a65dSAndrew Gallatin static char *mxge_fw_aligned = "mxge_eth_z8e"; 1161e413cf9SAndrew Gallatin static char *mxge_fw_rss_aligned = "mxge_rss_eth_z8e"; 1171e413cf9SAndrew Gallatin static char *mxge_fw_rss_unaligned = "mxge_rss_ethp_z8e"; 118b2fc195eSAndrew Gallatin 1196d87a65dSAndrew Gallatin static int mxge_probe(device_t dev); 1206d87a65dSAndrew Gallatin static int mxge_attach(device_t dev); 1216d87a65dSAndrew Gallatin static int mxge_detach(device_t dev); 1226d87a65dSAndrew Gallatin static int mxge_shutdown(device_t dev); 1236d87a65dSAndrew Gallatin static void mxge_intr(void *arg); 124b2fc195eSAndrew Gallatin 1256d87a65dSAndrew Gallatin static device_method_t mxge_methods[] = 126b2fc195eSAndrew Gallatin { 127b2fc195eSAndrew Gallatin /* Device interface */ 1286d87a65dSAndrew Gallatin DEVMETHOD(device_probe, mxge_probe), 1296d87a65dSAndrew Gallatin DEVMETHOD(device_attach, mxge_attach), 1306d87a65dSAndrew Gallatin DEVMETHOD(device_detach, mxge_detach), 1316d87a65dSAndrew Gallatin DEVMETHOD(device_shutdown, mxge_shutdown), 13261bfd867SSofian Brabez 13361bfd867SSofian Brabez DEVMETHOD_END 134b2fc195eSAndrew Gallatin }; 135b2fc195eSAndrew Gallatin 1366d87a65dSAndrew Gallatin static driver_t mxge_driver = 137b2fc195eSAndrew Gallatin { 1386d87a65dSAndrew Gallatin "mxge", 1396d87a65dSAndrew Gallatin mxge_methods, 1406d87a65dSAndrew Gallatin sizeof(mxge_softc_t), 141b2fc195eSAndrew Gallatin }; 142b2fc195eSAndrew Gallatin 1436d87a65dSAndrew Gallatin static devclass_t mxge_devclass; 144b2fc195eSAndrew Gallatin 145b2fc195eSAndrew Gallatin /* Declare ourselves to be a child of the PCI bus.*/ 1466d87a65dSAndrew Gallatin DRIVER_MODULE(mxge, pci, mxge_driver, mxge_devclass, 0, 0); 1476d87a65dSAndrew Gallatin MODULE_DEPEND(mxge, firmware, 1, 1, 1); 148f9ae0280SAndrew Gallatin MODULE_DEPEND(mxge, zlib, 1, 1, 1); 149b2fc195eSAndrew Gallatin 1501e413cf9SAndrew Gallatin static int mxge_load_firmware(mxge_softc_t *sc, int adopt); 1518fe615baSAndrew Gallatin static int mxge_send_cmd(mxge_softc_t *sc, uint32_t cmd, mxge_cmd_t *data); 152a393336bSAndrew Gallatin static int mxge_close(mxge_softc_t *sc, int down); 153276edd10SAndrew Gallatin static int mxge_open(mxge_softc_t *sc); 154276edd10SAndrew Gallatin static void mxge_tick(void *arg); 1558fe615baSAndrew Gallatin 156b2fc195eSAndrew Gallatin static int 1576d87a65dSAndrew Gallatin mxge_probe(device_t dev) 158b2fc195eSAndrew Gallatin { 15901638550SAndrew Gallatin int rev; 16001638550SAndrew Gallatin 16101638550SAndrew Gallatin 1626d87a65dSAndrew Gallatin if ((pci_get_vendor(dev) == MXGE_PCI_VENDOR_MYRICOM) && 163f1544498SAndrew Gallatin ((pci_get_device(dev) == MXGE_PCI_DEVICE_Z8E) || 164f1544498SAndrew Gallatin (pci_get_device(dev) == MXGE_PCI_DEVICE_Z8E_9))) { 16501638550SAndrew Gallatin rev = pci_get_revid(dev); 16601638550SAndrew Gallatin switch (rev) { 16701638550SAndrew Gallatin case MXGE_PCI_REV_Z8E: 168b2fc195eSAndrew Gallatin device_set_desc(dev, "Myri10G-PCIE-8A"); 16901638550SAndrew Gallatin break; 17001638550SAndrew Gallatin case MXGE_PCI_REV_Z8ES: 17101638550SAndrew Gallatin device_set_desc(dev, "Myri10G-PCIE-8B"); 17201638550SAndrew Gallatin break; 17301638550SAndrew Gallatin default: 17401638550SAndrew Gallatin device_set_desc(dev, "Myri10G-PCIE-8??"); 17501638550SAndrew Gallatin device_printf(dev, "Unrecognized rev %d NIC\n", 17601638550SAndrew Gallatin rev); 17701638550SAndrew Gallatin break; 17801638550SAndrew Gallatin } 179b2fc195eSAndrew Gallatin return 0; 180b2fc195eSAndrew Gallatin } 181b2fc195eSAndrew Gallatin return ENXIO; 182b2fc195eSAndrew Gallatin } 183b2fc195eSAndrew Gallatin 184b2fc195eSAndrew Gallatin static void 1856d87a65dSAndrew Gallatin mxge_enable_wc(mxge_softc_t *sc) 186b2fc195eSAndrew Gallatin { 187f9ae0280SAndrew Gallatin #if defined(__i386) || defined(__amd64) 188b2fc195eSAndrew Gallatin vm_offset_t len; 18947c2e987SAndrew Gallatin int err; 190b2fc195eSAndrew Gallatin 1914d69a9d0SAndrew Gallatin sc->wc = 1; 192b2fc195eSAndrew Gallatin len = rman_get_size(sc->mem_res); 193c2c14a69SAndrew Gallatin err = pmap_change_attr((vm_offset_t) sc->sram, 194c2c14a69SAndrew Gallatin len, PAT_WRITE_COMBINING); 19547c2e987SAndrew Gallatin if (err != 0) { 196c2c14a69SAndrew Gallatin device_printf(sc->dev, "pmap_change_attr failed, %d\n", 197c2c14a69SAndrew Gallatin err); 1984d69a9d0SAndrew Gallatin sc->wc = 0; 199b2fc195eSAndrew Gallatin } 200f9ae0280SAndrew Gallatin #endif 201b2fc195eSAndrew Gallatin } 202b2fc195eSAndrew Gallatin 203b2fc195eSAndrew Gallatin 204b2fc195eSAndrew Gallatin /* callback to get our DMA address */ 205b2fc195eSAndrew Gallatin static void 2066d87a65dSAndrew Gallatin mxge_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nsegs, 207b2fc195eSAndrew Gallatin int error) 208b2fc195eSAndrew Gallatin { 209b2fc195eSAndrew Gallatin if (error == 0) { 210b2fc195eSAndrew Gallatin *(bus_addr_t *) arg = segs->ds_addr; 211b2fc195eSAndrew Gallatin } 212b2fc195eSAndrew Gallatin } 213b2fc195eSAndrew Gallatin 214b2fc195eSAndrew Gallatin static int 2156d87a65dSAndrew Gallatin mxge_dma_alloc(mxge_softc_t *sc, mxge_dma_t *dma, size_t bytes, 216b2fc195eSAndrew Gallatin bus_size_t alignment) 217b2fc195eSAndrew Gallatin { 218b2fc195eSAndrew Gallatin int err; 219b2fc195eSAndrew Gallatin device_t dev = sc->dev; 2201e413cf9SAndrew Gallatin bus_size_t boundary, maxsegsize; 2211e413cf9SAndrew Gallatin 2221e413cf9SAndrew Gallatin if (bytes > 4096 && alignment == 4096) { 2231e413cf9SAndrew Gallatin boundary = 0; 2241e413cf9SAndrew Gallatin maxsegsize = bytes; 2251e413cf9SAndrew Gallatin } else { 2261e413cf9SAndrew Gallatin boundary = 4096; 2271e413cf9SAndrew Gallatin maxsegsize = 4096; 2281e413cf9SAndrew Gallatin } 229b2fc195eSAndrew Gallatin 230b2fc195eSAndrew Gallatin /* allocate DMAable memory tags */ 231b2fc195eSAndrew Gallatin err = bus_dma_tag_create(sc->parent_dmat, /* parent */ 232b2fc195eSAndrew Gallatin alignment, /* alignment */ 2331e413cf9SAndrew Gallatin boundary, /* boundary */ 234b2fc195eSAndrew Gallatin BUS_SPACE_MAXADDR, /* low */ 235b2fc195eSAndrew Gallatin BUS_SPACE_MAXADDR, /* high */ 236b2fc195eSAndrew Gallatin NULL, NULL, /* filter */ 237b2fc195eSAndrew Gallatin bytes, /* maxsize */ 238b2fc195eSAndrew Gallatin 1, /* num segs */ 2391e413cf9SAndrew Gallatin maxsegsize, /* maxsegsize */ 240b2fc195eSAndrew Gallatin BUS_DMA_COHERENT, /* flags */ 241b2fc195eSAndrew Gallatin NULL, NULL, /* lock */ 242b2fc195eSAndrew Gallatin &dma->dmat); /* tag */ 243b2fc195eSAndrew Gallatin if (err != 0) { 244b2fc195eSAndrew Gallatin device_printf(dev, "couldn't alloc tag (err = %d)\n", err); 245b2fc195eSAndrew Gallatin return err; 246b2fc195eSAndrew Gallatin } 247b2fc195eSAndrew Gallatin 248b2fc195eSAndrew Gallatin /* allocate DMAable memory & map */ 249b2fc195eSAndrew Gallatin err = bus_dmamem_alloc(dma->dmat, &dma->addr, 250b2fc195eSAndrew Gallatin (BUS_DMA_WAITOK | BUS_DMA_COHERENT 251b2fc195eSAndrew Gallatin | BUS_DMA_ZERO), &dma->map); 252b2fc195eSAndrew Gallatin if (err != 0) { 253b2fc195eSAndrew Gallatin device_printf(dev, "couldn't alloc mem (err = %d)\n", err); 254b2fc195eSAndrew Gallatin goto abort_with_dmat; 255b2fc195eSAndrew Gallatin } 256b2fc195eSAndrew Gallatin 257b2fc195eSAndrew Gallatin /* load the memory */ 258b2fc195eSAndrew Gallatin err = bus_dmamap_load(dma->dmat, dma->map, dma->addr, bytes, 2596d87a65dSAndrew Gallatin mxge_dmamap_callback, 260b2fc195eSAndrew Gallatin (void *)&dma->bus_addr, 0); 261b2fc195eSAndrew Gallatin if (err != 0) { 262b2fc195eSAndrew Gallatin device_printf(dev, "couldn't load map (err = %d)\n", err); 263b2fc195eSAndrew Gallatin goto abort_with_mem; 264b2fc195eSAndrew Gallatin } 265b2fc195eSAndrew Gallatin return 0; 266b2fc195eSAndrew Gallatin 267b2fc195eSAndrew Gallatin abort_with_mem: 268b2fc195eSAndrew Gallatin bus_dmamem_free(dma->dmat, dma->addr, dma->map); 269b2fc195eSAndrew Gallatin abort_with_dmat: 270b2fc195eSAndrew Gallatin (void)bus_dma_tag_destroy(dma->dmat); 271b2fc195eSAndrew Gallatin return err; 272b2fc195eSAndrew Gallatin } 273b2fc195eSAndrew Gallatin 274b2fc195eSAndrew Gallatin 275b2fc195eSAndrew Gallatin static void 2766d87a65dSAndrew Gallatin mxge_dma_free(mxge_dma_t *dma) 277b2fc195eSAndrew Gallatin { 278b2fc195eSAndrew Gallatin bus_dmamap_unload(dma->dmat, dma->map); 279b2fc195eSAndrew Gallatin bus_dmamem_free(dma->dmat, dma->addr, dma->map); 280b2fc195eSAndrew Gallatin (void)bus_dma_tag_destroy(dma->dmat); 281b2fc195eSAndrew Gallatin } 282b2fc195eSAndrew Gallatin 283b2fc195eSAndrew Gallatin /* 284b2fc195eSAndrew Gallatin * The eeprom strings on the lanaiX have the format 285b2fc195eSAndrew Gallatin * SN=x\0 286b2fc195eSAndrew Gallatin * MAC=x:x:x:x:x:x\0 287b2fc195eSAndrew Gallatin * PC=text\0 288b2fc195eSAndrew Gallatin */ 289b2fc195eSAndrew Gallatin 290b2fc195eSAndrew Gallatin static int 2916d87a65dSAndrew Gallatin mxge_parse_strings(mxge_softc_t *sc) 292b2fc195eSAndrew Gallatin { 293dedbe836SAndrew Gallatin char *ptr; 294a4b233ddSAndrew Gallatin int i, found_mac, found_sn2; 295dedbe836SAndrew Gallatin char *endptr; 296b2fc195eSAndrew Gallatin 297b2fc195eSAndrew Gallatin ptr = sc->eeprom_strings; 298b2fc195eSAndrew Gallatin found_mac = 0; 299a4b233ddSAndrew Gallatin found_sn2 = 0; 300dedbe836SAndrew Gallatin while (*ptr != '\0') { 301dedbe836SAndrew Gallatin if (strncmp(ptr, "MAC=", 4) == 0) { 302dedbe836SAndrew Gallatin ptr += 4; 303dedbe836SAndrew Gallatin for (i = 0;;) { 304dedbe836SAndrew Gallatin sc->mac_addr[i] = strtoul(ptr, &endptr, 16); 305dedbe836SAndrew Gallatin if (endptr - ptr != 2) 306b2fc195eSAndrew Gallatin goto abort; 307dedbe836SAndrew Gallatin ptr = endptr; 308dedbe836SAndrew Gallatin if (++i == 6) 309dedbe836SAndrew Gallatin break; 310dedbe836SAndrew Gallatin if (*ptr++ != ':') 311dedbe836SAndrew Gallatin goto abort; 312b2fc195eSAndrew Gallatin } 313dedbe836SAndrew Gallatin found_mac = 1; 314dedbe836SAndrew Gallatin } else if (strncmp(ptr, "PC=", 3) == 0) { 3155e7d8541SAndrew Gallatin ptr += 3; 316dedbe836SAndrew Gallatin strlcpy(sc->product_code_string, ptr, 317dedbe836SAndrew Gallatin sizeof(sc->product_code_string)); 318dedbe836SAndrew Gallatin } else if (!found_sn2 && (strncmp(ptr, "SN=", 3) == 0)) { 3195e7d8541SAndrew Gallatin ptr += 3; 320dedbe836SAndrew Gallatin strlcpy(sc->serial_number_string, ptr, 321dedbe836SAndrew Gallatin sizeof(sc->serial_number_string)); 322dedbe836SAndrew Gallatin } else if (strncmp(ptr, "SN2=", 4) == 0) { 323a4b233ddSAndrew Gallatin /* SN2 takes precedence over SN */ 324a4b233ddSAndrew Gallatin ptr += 4; 325a4b233ddSAndrew Gallatin found_sn2 = 1; 326dedbe836SAndrew Gallatin strlcpy(sc->serial_number_string, ptr, 327dedbe836SAndrew Gallatin sizeof(sc->serial_number_string)); 328b2fc195eSAndrew Gallatin } 329dedbe836SAndrew Gallatin while (*ptr++ != '\0') {} 330b2fc195eSAndrew Gallatin } 331b2fc195eSAndrew Gallatin 332b2fc195eSAndrew Gallatin if (found_mac) 333b2fc195eSAndrew Gallatin return 0; 334b2fc195eSAndrew Gallatin 335b2fc195eSAndrew Gallatin abort: 336b2fc195eSAndrew Gallatin device_printf(sc->dev, "failed to parse eeprom_strings\n"); 337b2fc195eSAndrew Gallatin 338b2fc195eSAndrew Gallatin return ENXIO; 339b2fc195eSAndrew Gallatin } 340b2fc195eSAndrew Gallatin 3410d151103SRoman Divacky #if defined __i386 || defined i386 || defined __i386__ || defined __x86_64__ 3428fe615baSAndrew Gallatin static void 3438fe615baSAndrew Gallatin mxge_enable_nvidia_ecrc(mxge_softc_t *sc) 344b2fc195eSAndrew Gallatin { 345b2fc195eSAndrew Gallatin uint32_t val; 3468fe615baSAndrew Gallatin unsigned long base, off; 347b2fc195eSAndrew Gallatin char *va, *cfgptr; 3488fe615baSAndrew Gallatin device_t pdev, mcp55; 3498fe615baSAndrew Gallatin uint16_t vendor_id, device_id, word; 350b2fc195eSAndrew Gallatin uintptr_t bus, slot, func, ivend, idev; 351b2fc195eSAndrew Gallatin uint32_t *ptr32; 352b2fc195eSAndrew Gallatin 3538fe615baSAndrew Gallatin 3548fe615baSAndrew Gallatin if (!mxge_nvidia_ecrc_enable) 3558fe615baSAndrew Gallatin return; 3568fe615baSAndrew Gallatin 3578fe615baSAndrew Gallatin pdev = device_get_parent(device_get_parent(sc->dev)); 3588fe615baSAndrew Gallatin if (pdev == NULL) { 3598fe615baSAndrew Gallatin device_printf(sc->dev, "could not find parent?\n"); 3608fe615baSAndrew Gallatin return; 3618fe615baSAndrew Gallatin } 3628fe615baSAndrew Gallatin vendor_id = pci_read_config(pdev, PCIR_VENDOR, 2); 3638fe615baSAndrew Gallatin device_id = pci_read_config(pdev, PCIR_DEVICE, 2); 3648fe615baSAndrew Gallatin 3658fe615baSAndrew Gallatin if (vendor_id != 0x10de) 3668fe615baSAndrew Gallatin return; 3678fe615baSAndrew Gallatin 3688fe615baSAndrew Gallatin base = 0; 3698fe615baSAndrew Gallatin 3708fe615baSAndrew Gallatin if (device_id == 0x005d) { 3718fe615baSAndrew Gallatin /* ck804, base address is magic */ 3728fe615baSAndrew Gallatin base = 0xe0000000UL; 3738fe615baSAndrew Gallatin } else if (device_id >= 0x0374 && device_id <= 0x378) { 3748fe615baSAndrew Gallatin /* mcp55, base address stored in chipset */ 3758fe615baSAndrew Gallatin mcp55 = pci_find_bsf(0, 0, 0); 3768fe615baSAndrew Gallatin if (mcp55 && 3778fe615baSAndrew Gallatin 0x10de == pci_read_config(mcp55, PCIR_VENDOR, 2) && 3788fe615baSAndrew Gallatin 0x0369 == pci_read_config(mcp55, PCIR_DEVICE, 2)) { 3798fe615baSAndrew Gallatin word = pci_read_config(mcp55, 0x90, 2); 3808fe615baSAndrew Gallatin base = ((unsigned long)word & 0x7ffeU) << 25; 3818fe615baSAndrew Gallatin } 3828fe615baSAndrew Gallatin } 3838fe615baSAndrew Gallatin if (!base) 3848fe615baSAndrew Gallatin return; 3858fe615baSAndrew Gallatin 386b2fc195eSAndrew Gallatin /* XXXX 387b2fc195eSAndrew Gallatin Test below is commented because it is believed that doing 388b2fc195eSAndrew Gallatin config read/write beyond 0xff will access the config space 389b2fc195eSAndrew Gallatin for the next larger function. Uncomment this and remove 390b2fc195eSAndrew Gallatin the hacky pmap_mapdev() way of accessing config space when 391b2fc195eSAndrew Gallatin FreeBSD grows support for extended pcie config space access 392b2fc195eSAndrew Gallatin */ 393b2fc195eSAndrew Gallatin #if 0 394b2fc195eSAndrew Gallatin /* See if we can, by some miracle, access the extended 395b2fc195eSAndrew Gallatin config space */ 396b2fc195eSAndrew Gallatin val = pci_read_config(pdev, 0x178, 4); 397b2fc195eSAndrew Gallatin if (val != 0xffffffff) { 398b2fc195eSAndrew Gallatin val |= 0x40; 399b2fc195eSAndrew Gallatin pci_write_config(pdev, 0x178, val, 4); 4008fe615baSAndrew Gallatin return; 401b2fc195eSAndrew Gallatin } 402b2fc195eSAndrew Gallatin #endif 403b2fc195eSAndrew Gallatin /* Rather than using normal pci config space writes, we must 404b2fc195eSAndrew Gallatin * map the Nvidia config space ourselves. This is because on 405b2fc195eSAndrew Gallatin * opteron/nvidia class machine the 0xe000000 mapping is 406b2fc195eSAndrew Gallatin * handled by the nvidia chipset, that means the internal PCI 407b2fc195eSAndrew Gallatin * device (the on-chip northbridge), or the amd-8131 bridge 408b2fc195eSAndrew Gallatin * and things behind them are not visible by this method. 409b2fc195eSAndrew Gallatin */ 410b2fc195eSAndrew Gallatin 411b2fc195eSAndrew Gallatin BUS_READ_IVAR(device_get_parent(pdev), pdev, 412b2fc195eSAndrew Gallatin PCI_IVAR_BUS, &bus); 413b2fc195eSAndrew Gallatin BUS_READ_IVAR(device_get_parent(pdev), pdev, 414b2fc195eSAndrew Gallatin PCI_IVAR_SLOT, &slot); 415b2fc195eSAndrew Gallatin BUS_READ_IVAR(device_get_parent(pdev), pdev, 416b2fc195eSAndrew Gallatin PCI_IVAR_FUNCTION, &func); 417b2fc195eSAndrew Gallatin BUS_READ_IVAR(device_get_parent(pdev), pdev, 418b2fc195eSAndrew Gallatin PCI_IVAR_VENDOR, &ivend); 419b2fc195eSAndrew Gallatin BUS_READ_IVAR(device_get_parent(pdev), pdev, 420b2fc195eSAndrew Gallatin PCI_IVAR_DEVICE, &idev); 421b2fc195eSAndrew Gallatin 4228fe615baSAndrew Gallatin off = base 423b2fc195eSAndrew Gallatin + 0x00100000UL * (unsigned long)bus 424b2fc195eSAndrew Gallatin + 0x00001000UL * (unsigned long)(func 425b2fc195eSAndrew Gallatin + 8 * slot); 426b2fc195eSAndrew Gallatin 427b2fc195eSAndrew Gallatin /* map it into the kernel */ 428b2fc195eSAndrew Gallatin va = pmap_mapdev(trunc_page((vm_paddr_t)off), PAGE_SIZE); 429b2fc195eSAndrew Gallatin 430b2fc195eSAndrew Gallatin 431b2fc195eSAndrew Gallatin if (va == NULL) { 432b2fc195eSAndrew Gallatin device_printf(sc->dev, "pmap_kenter_temporary didn't\n"); 4338fe615baSAndrew Gallatin return; 434b2fc195eSAndrew Gallatin } 435b2fc195eSAndrew Gallatin /* get a pointer to the config space mapped into the kernel */ 436b2fc195eSAndrew Gallatin cfgptr = va + (off & PAGE_MASK); 437b2fc195eSAndrew Gallatin 438b2fc195eSAndrew Gallatin /* make sure that we can really access it */ 439b2fc195eSAndrew Gallatin vendor_id = *(uint16_t *)(cfgptr + PCIR_VENDOR); 440b2fc195eSAndrew Gallatin device_id = *(uint16_t *)(cfgptr + PCIR_DEVICE); 441b2fc195eSAndrew Gallatin if (! (vendor_id == ivend && device_id == idev)) { 442b2fc195eSAndrew Gallatin device_printf(sc->dev, "mapping failed: 0x%x:0x%x\n", 443b2fc195eSAndrew Gallatin vendor_id, device_id); 444b2fc195eSAndrew Gallatin pmap_unmapdev((vm_offset_t)va, PAGE_SIZE); 4458fe615baSAndrew Gallatin return; 446b2fc195eSAndrew Gallatin } 447b2fc195eSAndrew Gallatin 448b2fc195eSAndrew Gallatin ptr32 = (uint32_t*)(cfgptr + 0x178); 449b2fc195eSAndrew Gallatin val = *ptr32; 450b2fc195eSAndrew Gallatin 451b2fc195eSAndrew Gallatin if (val == 0xffffffff) { 452b2fc195eSAndrew Gallatin device_printf(sc->dev, "extended mapping failed\n"); 453b2fc195eSAndrew Gallatin pmap_unmapdev((vm_offset_t)va, PAGE_SIZE); 4548fe615baSAndrew Gallatin return; 455b2fc195eSAndrew Gallatin } 456b2fc195eSAndrew Gallatin *ptr32 = val | 0x40; 457b2fc195eSAndrew Gallatin pmap_unmapdev((vm_offset_t)va, PAGE_SIZE); 4585e7d8541SAndrew Gallatin if (mxge_verbose) 459b2fc195eSAndrew Gallatin device_printf(sc->dev, 4605e7d8541SAndrew Gallatin "Enabled ECRC on upstream Nvidia bridge " 4615e7d8541SAndrew Gallatin "at %d:%d:%d\n", 462b2fc195eSAndrew Gallatin (int)bus, (int)slot, (int)func); 4638fe615baSAndrew Gallatin return; 464b2fc195eSAndrew Gallatin } 465b2fc195eSAndrew Gallatin #else 4668fe615baSAndrew Gallatin static void 467f9ae0280SAndrew Gallatin mxge_enable_nvidia_ecrc(mxge_softc_t *sc) 468b2fc195eSAndrew Gallatin { 469b2fc195eSAndrew Gallatin device_printf(sc->dev, 470b2fc195eSAndrew Gallatin "Nforce 4 chipset on non-x86/amd64!?!?!\n"); 4718fe615baSAndrew Gallatin return; 472b2fc195eSAndrew Gallatin } 473b2fc195eSAndrew Gallatin #endif 4748fe615baSAndrew Gallatin 4758fe615baSAndrew Gallatin 4768fe615baSAndrew Gallatin static int 4778fe615baSAndrew Gallatin mxge_dma_test(mxge_softc_t *sc, int test_type) 4788fe615baSAndrew Gallatin { 4798fe615baSAndrew Gallatin mxge_cmd_t cmd; 4808fe615baSAndrew Gallatin bus_addr_t dmatest_bus = sc->dmabench_dma.bus_addr; 4818fe615baSAndrew Gallatin int status; 4828fe615baSAndrew Gallatin uint32_t len; 4838fe615baSAndrew Gallatin char *test = " "; 4848fe615baSAndrew Gallatin 4858fe615baSAndrew Gallatin 4868fe615baSAndrew Gallatin /* Run a small DMA test. 4878fe615baSAndrew Gallatin * The magic multipliers to the length tell the firmware 4888fe615baSAndrew Gallatin * to do DMA read, write, or read+write tests. The 4898fe615baSAndrew Gallatin * results are returned in cmd.data0. The upper 16 4908fe615baSAndrew Gallatin * bits of the return is the number of transfers completed. 4918fe615baSAndrew Gallatin * The lower 16 bits is the time in 0.5us ticks that the 4928fe615baSAndrew Gallatin * transfers took to complete. 4938fe615baSAndrew Gallatin */ 4948fe615baSAndrew Gallatin 4951e413cf9SAndrew Gallatin len = sc->tx_boundary; 4968fe615baSAndrew Gallatin 4978fe615baSAndrew Gallatin cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus); 4988fe615baSAndrew Gallatin cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus); 4998fe615baSAndrew Gallatin cmd.data2 = len * 0x10000; 5008fe615baSAndrew Gallatin status = mxge_send_cmd(sc, test_type, &cmd); 5018fe615baSAndrew Gallatin if (status != 0) { 5028fe615baSAndrew Gallatin test = "read"; 5038fe615baSAndrew Gallatin goto abort; 5048fe615baSAndrew Gallatin } 5058fe615baSAndrew Gallatin sc->read_dma = ((cmd.data0>>16) * len * 2) / 5068fe615baSAndrew Gallatin (cmd.data0 & 0xffff); 5078fe615baSAndrew Gallatin cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus); 5088fe615baSAndrew Gallatin cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus); 5098fe615baSAndrew Gallatin cmd.data2 = len * 0x1; 5108fe615baSAndrew Gallatin status = mxge_send_cmd(sc, test_type, &cmd); 5118fe615baSAndrew Gallatin if (status != 0) { 5128fe615baSAndrew Gallatin test = "write"; 5138fe615baSAndrew Gallatin goto abort; 5148fe615baSAndrew Gallatin } 5158fe615baSAndrew Gallatin sc->write_dma = ((cmd.data0>>16) * len * 2) / 5168fe615baSAndrew Gallatin (cmd.data0 & 0xffff); 5178fe615baSAndrew Gallatin 5188fe615baSAndrew Gallatin cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus); 5198fe615baSAndrew Gallatin cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus); 5208fe615baSAndrew Gallatin cmd.data2 = len * 0x10001; 5218fe615baSAndrew Gallatin status = mxge_send_cmd(sc, test_type, &cmd); 5228fe615baSAndrew Gallatin if (status != 0) { 5238fe615baSAndrew Gallatin test = "read/write"; 5248fe615baSAndrew Gallatin goto abort; 5258fe615baSAndrew Gallatin } 5268fe615baSAndrew Gallatin sc->read_write_dma = ((cmd.data0>>16) * len * 2 * 2) / 5278fe615baSAndrew Gallatin (cmd.data0 & 0xffff); 5288fe615baSAndrew Gallatin 5298fe615baSAndrew Gallatin abort: 5308fe615baSAndrew Gallatin if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST) 5318fe615baSAndrew Gallatin device_printf(sc->dev, "DMA %s benchmark failed: %d\n", 5328fe615baSAndrew Gallatin test, status); 5338fe615baSAndrew Gallatin 5348fe615baSAndrew Gallatin return status; 5358fe615baSAndrew Gallatin } 5368fe615baSAndrew Gallatin 537b2fc195eSAndrew Gallatin /* 538b2fc195eSAndrew Gallatin * The Lanai Z8E PCI-E interface achieves higher Read-DMA throughput 539b2fc195eSAndrew Gallatin * when the PCI-E Completion packets are aligned on an 8-byte 540b2fc195eSAndrew Gallatin * boundary. Some PCI-E chip sets always align Completion packets; on 541b2fc195eSAndrew Gallatin * the ones that do not, the alignment can be enforced by enabling 542b2fc195eSAndrew Gallatin * ECRC generation (if supported). 543b2fc195eSAndrew Gallatin * 544b2fc195eSAndrew Gallatin * When PCI-E Completion packets are not aligned, it is actually more 545b2fc195eSAndrew Gallatin * efficient to limit Read-DMA transactions to 2KB, rather than 4KB. 546b2fc195eSAndrew Gallatin * 547b2fc195eSAndrew Gallatin * If the driver can neither enable ECRC nor verify that it has 548b2fc195eSAndrew Gallatin * already been enabled, then it must use a firmware image which works 549b2fc195eSAndrew Gallatin * around unaligned completion packets (ethp_z8e.dat), and it should 550b2fc195eSAndrew Gallatin * also ensure that it never gives the device a Read-DMA which is 5511e413cf9SAndrew Gallatin * larger than 2KB by setting the tx_boundary to 2KB. If ECRC is 552b2fc195eSAndrew Gallatin * enabled, then the driver should use the aligned (eth_z8e.dat) 5531e413cf9SAndrew Gallatin * firmware image, and set tx_boundary to 4KB. 554b2fc195eSAndrew Gallatin */ 555b2fc195eSAndrew Gallatin 5568fe615baSAndrew Gallatin static int 5578fe615baSAndrew Gallatin mxge_firmware_probe(mxge_softc_t *sc) 5588fe615baSAndrew Gallatin { 5598fe615baSAndrew Gallatin device_t dev = sc->dev; 5608fe615baSAndrew Gallatin int reg, status; 5618fe615baSAndrew Gallatin uint16_t pectl; 5628fe615baSAndrew Gallatin 5631e413cf9SAndrew Gallatin sc->tx_boundary = 4096; 5648fe615baSAndrew Gallatin /* 5658fe615baSAndrew Gallatin * Verify the max read request size was set to 4KB 5668fe615baSAndrew Gallatin * before trying the test with 4KB. 5678fe615baSAndrew Gallatin */ 5683b0a4aefSJohn Baldwin if (pci_find_cap(dev, PCIY_EXPRESS, ®) == 0) { 5698fe615baSAndrew Gallatin pectl = pci_read_config(dev, reg + 0x8, 2); 5708fe615baSAndrew Gallatin if ((pectl & (5 << 12)) != (5 << 12)) { 5718fe615baSAndrew Gallatin device_printf(dev, "Max Read Req. size != 4k (0x%x\n", 5728fe615baSAndrew Gallatin pectl); 5731e413cf9SAndrew Gallatin sc->tx_boundary = 2048; 5748fe615baSAndrew Gallatin } 5758fe615baSAndrew Gallatin } 5768fe615baSAndrew Gallatin 5778fe615baSAndrew Gallatin /* 5788fe615baSAndrew Gallatin * load the optimized firmware (which assumes aligned PCIe 5798fe615baSAndrew Gallatin * completions) in order to see if it works on this host. 5808fe615baSAndrew Gallatin */ 5818fe615baSAndrew Gallatin sc->fw_name = mxge_fw_aligned; 5821e413cf9SAndrew Gallatin status = mxge_load_firmware(sc, 1); 5838fe615baSAndrew Gallatin if (status != 0) { 5848fe615baSAndrew Gallatin return status; 5858fe615baSAndrew Gallatin } 5868fe615baSAndrew Gallatin 5878fe615baSAndrew Gallatin /* 5888fe615baSAndrew Gallatin * Enable ECRC if possible 5898fe615baSAndrew Gallatin */ 5908fe615baSAndrew Gallatin mxge_enable_nvidia_ecrc(sc); 5918fe615baSAndrew Gallatin 5928fe615baSAndrew Gallatin /* 5938fe615baSAndrew Gallatin * Run a DMA test which watches for unaligned completions and 594a4b233ddSAndrew Gallatin * aborts on the first one seen. Not required on Z8ES or newer. 5958fe615baSAndrew Gallatin */ 596a4b233ddSAndrew Gallatin if (pci_get_revid(sc->dev) >= MXGE_PCI_REV_Z8ES) 597a4b233ddSAndrew Gallatin return 0; 5988fe615baSAndrew Gallatin status = mxge_dma_test(sc, MXGEFW_CMD_UNALIGNED_TEST); 5998fe615baSAndrew Gallatin if (status == 0) 6008fe615baSAndrew Gallatin return 0; /* keep the aligned firmware */ 6018fe615baSAndrew Gallatin 6028fe615baSAndrew Gallatin if (status != E2BIG) 6038fe615baSAndrew Gallatin device_printf(dev, "DMA test failed: %d\n", status); 6048fe615baSAndrew Gallatin if (status == ENOSYS) 6058fe615baSAndrew Gallatin device_printf(dev, "Falling back to ethp! " 6068fe615baSAndrew Gallatin "Please install up to date fw\n"); 6078fe615baSAndrew Gallatin return status; 6088fe615baSAndrew Gallatin } 6098fe615baSAndrew Gallatin 6108fe615baSAndrew Gallatin static int 6116d87a65dSAndrew Gallatin mxge_select_firmware(mxge_softc_t *sc) 612b2fc195eSAndrew Gallatin { 6138fe615baSAndrew Gallatin int aligned = 0; 61465c69066SAndrew Gallatin int force_firmware = mxge_force_firmware; 615b2fc195eSAndrew Gallatin 61665c69066SAndrew Gallatin if (sc->throttle) 61765c69066SAndrew Gallatin force_firmware = sc->throttle; 618d91b1b49SAndrew Gallatin 61965c69066SAndrew Gallatin if (force_firmware != 0) { 62065c69066SAndrew Gallatin if (force_firmware == 1) 621d91b1b49SAndrew Gallatin aligned = 1; 622d91b1b49SAndrew Gallatin else 623d91b1b49SAndrew Gallatin aligned = 0; 624d91b1b49SAndrew Gallatin if (mxge_verbose) 625d91b1b49SAndrew Gallatin device_printf(sc->dev, 626d91b1b49SAndrew Gallatin "Assuming %s completions (forced)\n", 627d91b1b49SAndrew Gallatin aligned ? "aligned" : "unaligned"); 628d91b1b49SAndrew Gallatin goto abort; 629d91b1b49SAndrew Gallatin } 630d91b1b49SAndrew Gallatin 631d91b1b49SAndrew Gallatin /* if the PCIe link width is 4 or less, we can use the aligned 632d91b1b49SAndrew Gallatin firmware and skip any checks */ 633d91b1b49SAndrew Gallatin if (sc->link_width != 0 && sc->link_width <= 4) { 634d91b1b49SAndrew Gallatin device_printf(sc->dev, 635d91b1b49SAndrew Gallatin "PCIe x%d Link, expect reduced performance\n", 636d91b1b49SAndrew Gallatin sc->link_width); 637d91b1b49SAndrew Gallatin aligned = 1; 638d91b1b49SAndrew Gallatin goto abort; 639d91b1b49SAndrew Gallatin } 640d91b1b49SAndrew Gallatin 6418fe615baSAndrew Gallatin if (0 == mxge_firmware_probe(sc)) 6428fe615baSAndrew Gallatin return 0; 643b2fc195eSAndrew Gallatin 644b2fc195eSAndrew Gallatin abort: 645b2fc195eSAndrew Gallatin if (aligned) { 6466d87a65dSAndrew Gallatin sc->fw_name = mxge_fw_aligned; 6471e413cf9SAndrew Gallatin sc->tx_boundary = 4096; 648b2fc195eSAndrew Gallatin } else { 6496d87a65dSAndrew Gallatin sc->fw_name = mxge_fw_unaligned; 6501e413cf9SAndrew Gallatin sc->tx_boundary = 2048; 651b2fc195eSAndrew Gallatin } 6521e413cf9SAndrew Gallatin return (mxge_load_firmware(sc, 0)); 653b2fc195eSAndrew Gallatin } 654b2fc195eSAndrew Gallatin 6554da0d523SAndrew Gallatin static int 6564da0d523SAndrew Gallatin mxge_validate_firmware(mxge_softc_t *sc, const mcp_gen_header_t *hdr) 6574da0d523SAndrew Gallatin { 658b824b7d8SAndrew Gallatin 6594da0d523SAndrew Gallatin 6604da0d523SAndrew Gallatin if (be32toh(hdr->mcp_type) != MCP_TYPE_ETH) { 6614da0d523SAndrew Gallatin device_printf(sc->dev, "Bad firmware type: 0x%x\n", 6624da0d523SAndrew Gallatin be32toh(hdr->mcp_type)); 6634da0d523SAndrew Gallatin return EIO; 6644da0d523SAndrew Gallatin } 6654da0d523SAndrew Gallatin 6664da0d523SAndrew Gallatin /* save firmware version for sysctl */ 667dedbe836SAndrew Gallatin strlcpy(sc->fw_version, hdr->version, sizeof(sc->fw_version)); 6684da0d523SAndrew Gallatin if (mxge_verbose) 6694da0d523SAndrew Gallatin device_printf(sc->dev, "firmware id: %s\n", hdr->version); 6704da0d523SAndrew Gallatin 671b824b7d8SAndrew Gallatin sscanf(sc->fw_version, "%d.%d.%d", &sc->fw_ver_major, 672b824b7d8SAndrew Gallatin &sc->fw_ver_minor, &sc->fw_ver_tiny); 6734da0d523SAndrew Gallatin 674b824b7d8SAndrew Gallatin if (!(sc->fw_ver_major == MXGEFW_VERSION_MAJOR 675b824b7d8SAndrew Gallatin && sc->fw_ver_minor == MXGEFW_VERSION_MINOR)) { 6764da0d523SAndrew Gallatin device_printf(sc->dev, "Found firmware version %s\n", 6774da0d523SAndrew Gallatin sc->fw_version); 6784da0d523SAndrew Gallatin device_printf(sc->dev, "Driver needs %d.%d\n", 6794da0d523SAndrew Gallatin MXGEFW_VERSION_MAJOR, MXGEFW_VERSION_MINOR); 6804da0d523SAndrew Gallatin return EINVAL; 6814da0d523SAndrew Gallatin } 6824da0d523SAndrew Gallatin return 0; 6834da0d523SAndrew Gallatin 6844da0d523SAndrew Gallatin } 685b2fc195eSAndrew Gallatin 686f9ae0280SAndrew Gallatin static void * 687f9ae0280SAndrew Gallatin z_alloc(void *nil, u_int items, u_int size) 688f9ae0280SAndrew Gallatin { 689f9ae0280SAndrew Gallatin void *ptr; 690f9ae0280SAndrew Gallatin 691ac2fffa4SPedro F. Giffuni ptr = malloc(items * size, M_TEMP, M_NOWAIT); 692f9ae0280SAndrew Gallatin return ptr; 693f9ae0280SAndrew Gallatin } 694f9ae0280SAndrew Gallatin 695f9ae0280SAndrew Gallatin static void 696f9ae0280SAndrew Gallatin z_free(void *nil, void *ptr) 697f9ae0280SAndrew Gallatin { 698f9ae0280SAndrew Gallatin free(ptr, M_TEMP); 699f9ae0280SAndrew Gallatin } 700f9ae0280SAndrew Gallatin 701f9ae0280SAndrew Gallatin 702b2fc195eSAndrew Gallatin static int 7036d87a65dSAndrew Gallatin mxge_load_firmware_helper(mxge_softc_t *sc, uint32_t *limit) 704b2fc195eSAndrew Gallatin { 705f9ae0280SAndrew Gallatin z_stream zs; 706f9ae0280SAndrew Gallatin char *inflate_buffer; 70733d54970SLuigi Rizzo const struct firmware *fw; 708b2fc195eSAndrew Gallatin const mcp_gen_header_t *hdr; 709b2fc195eSAndrew Gallatin unsigned hdr_offset; 710b2fc195eSAndrew Gallatin int status; 7114da0d523SAndrew Gallatin unsigned int i; 7124da0d523SAndrew Gallatin char dummy; 713f9ae0280SAndrew Gallatin size_t fw_len; 714b2fc195eSAndrew Gallatin 715b2fc195eSAndrew Gallatin fw = firmware_get(sc->fw_name); 716b2fc195eSAndrew Gallatin if (fw == NULL) { 717b2fc195eSAndrew Gallatin device_printf(sc->dev, "Could not find firmware image %s\n", 718b2fc195eSAndrew Gallatin sc->fw_name); 719b2fc195eSAndrew Gallatin return ENOENT; 720b2fc195eSAndrew Gallatin } 721b2fc195eSAndrew Gallatin 722f9ae0280SAndrew Gallatin 723f9ae0280SAndrew Gallatin 724f9ae0280SAndrew Gallatin /* setup zlib and decompress f/w */ 725f9ae0280SAndrew Gallatin bzero(&zs, sizeof (zs)); 726f9ae0280SAndrew Gallatin zs.zalloc = z_alloc; 727f9ae0280SAndrew Gallatin zs.zfree = z_free; 728f9ae0280SAndrew Gallatin status = inflateInit(&zs); 729f9ae0280SAndrew Gallatin if (status != Z_OK) { 730b2fc195eSAndrew Gallatin status = EIO; 731b2fc195eSAndrew Gallatin goto abort_with_fw; 732b2fc195eSAndrew Gallatin } 733f9ae0280SAndrew Gallatin 734f9ae0280SAndrew Gallatin /* the uncompressed size is stored as the firmware version, 735f9ae0280SAndrew Gallatin which would otherwise go unused */ 736f9ae0280SAndrew Gallatin fw_len = (size_t) fw->version; 737f9ae0280SAndrew Gallatin inflate_buffer = malloc(fw_len, M_TEMP, M_NOWAIT); 738f9ae0280SAndrew Gallatin if (inflate_buffer == NULL) 739f9ae0280SAndrew Gallatin goto abort_with_zs; 740f9ae0280SAndrew Gallatin zs.avail_in = fw->datasize; 741f9ae0280SAndrew Gallatin zs.next_in = __DECONST(char *, fw->data); 742f9ae0280SAndrew Gallatin zs.avail_out = fw_len; 743f9ae0280SAndrew Gallatin zs.next_out = inflate_buffer; 744f9ae0280SAndrew Gallatin status = inflate(&zs, Z_FINISH); 745f9ae0280SAndrew Gallatin if (status != Z_STREAM_END) { 746f9ae0280SAndrew Gallatin device_printf(sc->dev, "zlib %d\n", status); 747f9ae0280SAndrew Gallatin status = EIO; 748f9ae0280SAndrew Gallatin goto abort_with_buffer; 749f9ae0280SAndrew Gallatin } 750f9ae0280SAndrew Gallatin 751f9ae0280SAndrew Gallatin /* check id */ 752f9ae0280SAndrew Gallatin hdr_offset = htobe32(*(const uint32_t *) 753f9ae0280SAndrew Gallatin (inflate_buffer + MCP_HEADER_PTR_OFFSET)); 754f9ae0280SAndrew Gallatin if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > fw_len) { 755f9ae0280SAndrew Gallatin device_printf(sc->dev, "Bad firmware file"); 756f9ae0280SAndrew Gallatin status = EIO; 757f9ae0280SAndrew Gallatin goto abort_with_buffer; 758f9ae0280SAndrew Gallatin } 759f9ae0280SAndrew Gallatin hdr = (const void*)(inflate_buffer + hdr_offset); 760b2fc195eSAndrew Gallatin 7614da0d523SAndrew Gallatin status = mxge_validate_firmware(sc, hdr); 7624da0d523SAndrew Gallatin if (status != 0) 763f9ae0280SAndrew Gallatin goto abort_with_buffer; 764b2fc195eSAndrew Gallatin 765b2fc195eSAndrew Gallatin /* Copy the inflated firmware to NIC SRAM. */ 766f9ae0280SAndrew Gallatin for (i = 0; i < fw_len; i += 256) { 7674da0d523SAndrew Gallatin mxge_pio_copy(sc->sram + MXGE_FW_OFFSET + i, 768f9ae0280SAndrew Gallatin inflate_buffer + i, 769f9ae0280SAndrew Gallatin min(256U, (unsigned)(fw_len - i))); 77073c7c83fSAndrew Gallatin wmb(); 7714da0d523SAndrew Gallatin dummy = *sc->sram; 77273c7c83fSAndrew Gallatin wmb(); 7734da0d523SAndrew Gallatin } 774b2fc195eSAndrew Gallatin 775f9ae0280SAndrew Gallatin *limit = fw_len; 776b2fc195eSAndrew Gallatin status = 0; 777f9ae0280SAndrew Gallatin abort_with_buffer: 778f9ae0280SAndrew Gallatin free(inflate_buffer, M_TEMP); 779f9ae0280SAndrew Gallatin abort_with_zs: 780f9ae0280SAndrew Gallatin inflateEnd(&zs); 781b2fc195eSAndrew Gallatin abort_with_fw: 782b2fc195eSAndrew Gallatin firmware_put(fw, FIRMWARE_UNLOAD); 783b2fc195eSAndrew Gallatin return status; 784b2fc195eSAndrew Gallatin } 785b2fc195eSAndrew Gallatin 786b2fc195eSAndrew Gallatin /* 787b2fc195eSAndrew Gallatin * Enable or disable periodic RDMAs from the host to make certain 788b2fc195eSAndrew Gallatin * chipsets resend dropped PCIe messages 789b2fc195eSAndrew Gallatin */ 790b2fc195eSAndrew Gallatin 791b2fc195eSAndrew Gallatin static void 7926d87a65dSAndrew Gallatin mxge_dummy_rdma(mxge_softc_t *sc, int enable) 793b2fc195eSAndrew Gallatin { 794b2fc195eSAndrew Gallatin char buf_bytes[72]; 795b2fc195eSAndrew Gallatin volatile uint32_t *confirm; 796b2fc195eSAndrew Gallatin volatile char *submit; 797b2fc195eSAndrew Gallatin uint32_t *buf, dma_low, dma_high; 798b2fc195eSAndrew Gallatin int i; 799b2fc195eSAndrew Gallatin 800b2fc195eSAndrew Gallatin buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL); 801b2fc195eSAndrew Gallatin 802b2fc195eSAndrew Gallatin /* clear confirmation addr */ 803b2fc195eSAndrew Gallatin confirm = (volatile uint32_t *)sc->cmd; 804b2fc195eSAndrew Gallatin *confirm = 0; 80573c7c83fSAndrew Gallatin wmb(); 806b2fc195eSAndrew Gallatin 807b2fc195eSAndrew Gallatin /* send an rdma command to the PCIe engine, and wait for the 808b2fc195eSAndrew Gallatin response in the confirmation address. The firmware should 809b2fc195eSAndrew Gallatin write a -1 there to indicate it is alive and well 810b2fc195eSAndrew Gallatin */ 811b2fc195eSAndrew Gallatin 8126d87a65dSAndrew Gallatin dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr); 8136d87a65dSAndrew Gallatin dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr); 814b2fc195eSAndrew Gallatin buf[0] = htobe32(dma_high); /* confirm addr MSW */ 815b2fc195eSAndrew Gallatin buf[1] = htobe32(dma_low); /* confirm addr LSW */ 816b2fc195eSAndrew Gallatin buf[2] = htobe32(0xffffffff); /* confirm data */ 8176d87a65dSAndrew Gallatin dma_low = MXGE_LOWPART_TO_U32(sc->zeropad_dma.bus_addr); 8186d87a65dSAndrew Gallatin dma_high = MXGE_HIGHPART_TO_U32(sc->zeropad_dma.bus_addr); 819b2fc195eSAndrew Gallatin buf[3] = htobe32(dma_high); /* dummy addr MSW */ 820b2fc195eSAndrew Gallatin buf[4] = htobe32(dma_low); /* dummy addr LSW */ 821b2fc195eSAndrew Gallatin buf[5] = htobe32(enable); /* enable? */ 822b2fc195eSAndrew Gallatin 823b2fc195eSAndrew Gallatin 8240fa7f681SAndrew Gallatin submit = (volatile char *)(sc->sram + MXGEFW_BOOT_DUMMY_RDMA); 825b2fc195eSAndrew Gallatin 8266d87a65dSAndrew Gallatin mxge_pio_copy(submit, buf, 64); 82773c7c83fSAndrew Gallatin wmb(); 828b2fc195eSAndrew Gallatin DELAY(1000); 82973c7c83fSAndrew Gallatin wmb(); 830b2fc195eSAndrew Gallatin i = 0; 831b2fc195eSAndrew Gallatin while (*confirm != 0xffffffff && i < 20) { 832b2fc195eSAndrew Gallatin DELAY(1000); 833b2fc195eSAndrew Gallatin i++; 834b2fc195eSAndrew Gallatin } 835b2fc195eSAndrew Gallatin if (*confirm != 0xffffffff) { 836b2fc195eSAndrew Gallatin device_printf(sc->dev, "dummy rdma %s failed (%p = 0x%x)", 837b2fc195eSAndrew Gallatin (enable ? "enable" : "disable"), confirm, 838b2fc195eSAndrew Gallatin *confirm); 839b2fc195eSAndrew Gallatin } 840b2fc195eSAndrew Gallatin return; 841b2fc195eSAndrew Gallatin } 842b2fc195eSAndrew Gallatin 843b2fc195eSAndrew Gallatin static int 8446d87a65dSAndrew Gallatin mxge_send_cmd(mxge_softc_t *sc, uint32_t cmd, mxge_cmd_t *data) 845b2fc195eSAndrew Gallatin { 846b2fc195eSAndrew Gallatin mcp_cmd_t *buf; 847b2fc195eSAndrew Gallatin char buf_bytes[sizeof(*buf) + 8]; 848b2fc195eSAndrew Gallatin volatile mcp_cmd_response_t *response = sc->cmd; 8490fa7f681SAndrew Gallatin volatile char *cmd_addr = sc->sram + MXGEFW_ETH_CMD; 850b2fc195eSAndrew Gallatin uint32_t dma_low, dma_high; 851e0501fd0SAndrew Gallatin int err, sleep_total = 0; 852b2fc195eSAndrew Gallatin 853b2fc195eSAndrew Gallatin /* ensure buf is aligned to 8 bytes */ 854b2fc195eSAndrew Gallatin buf = (mcp_cmd_t *)((unsigned long)(buf_bytes + 7) & ~7UL); 855b2fc195eSAndrew Gallatin 856b2fc195eSAndrew Gallatin buf->data0 = htobe32(data->data0); 857b2fc195eSAndrew Gallatin buf->data1 = htobe32(data->data1); 858b2fc195eSAndrew Gallatin buf->data2 = htobe32(data->data2); 859b2fc195eSAndrew Gallatin buf->cmd = htobe32(cmd); 8606d87a65dSAndrew Gallatin dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr); 8616d87a65dSAndrew Gallatin dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr); 862b2fc195eSAndrew Gallatin 863b2fc195eSAndrew Gallatin buf->response_addr.low = htobe32(dma_low); 864b2fc195eSAndrew Gallatin buf->response_addr.high = htobe32(dma_high); 865a98d6cd7SAndrew Gallatin mtx_lock(&sc->cmd_mtx); 866b2fc195eSAndrew Gallatin response->result = 0xffffffff; 86773c7c83fSAndrew Gallatin wmb(); 8686d87a65dSAndrew Gallatin mxge_pio_copy((volatile void *)cmd_addr, buf, sizeof (*buf)); 869b2fc195eSAndrew Gallatin 8705e7d8541SAndrew Gallatin /* wait up to 20ms */ 871e0501fd0SAndrew Gallatin err = EAGAIN; 8725e7d8541SAndrew Gallatin for (sleep_total = 0; sleep_total < 20; sleep_total++) { 873b2fc195eSAndrew Gallatin bus_dmamap_sync(sc->cmd_dma.dmat, 874b2fc195eSAndrew Gallatin sc->cmd_dma.map, BUS_DMASYNC_POSTREAD); 87573c7c83fSAndrew Gallatin wmb(); 876e0501fd0SAndrew Gallatin switch (be32toh(response->result)) { 877e0501fd0SAndrew Gallatin case 0: 878b2fc195eSAndrew Gallatin data->data0 = be32toh(response->data); 879e0501fd0SAndrew Gallatin err = 0; 880e0501fd0SAndrew Gallatin break; 881e0501fd0SAndrew Gallatin case 0xffffffff: 882e0501fd0SAndrew Gallatin DELAY(1000); 883e0501fd0SAndrew Gallatin break; 884e0501fd0SAndrew Gallatin case MXGEFW_CMD_UNKNOWN: 885e0501fd0SAndrew Gallatin err = ENOSYS; 886e0501fd0SAndrew Gallatin break; 887e0501fd0SAndrew Gallatin case MXGEFW_CMD_ERROR_UNALIGNED: 888e0501fd0SAndrew Gallatin err = E2BIG; 889e0501fd0SAndrew Gallatin break; 890c587e59fSAndrew Gallatin case MXGEFW_CMD_ERROR_BUSY: 891c587e59fSAndrew Gallatin err = EBUSY; 892c587e59fSAndrew Gallatin break; 893c406ad2eSAndrew Gallatin case MXGEFW_CMD_ERROR_I2C_ABSENT: 894c406ad2eSAndrew Gallatin err = ENXIO; 895c406ad2eSAndrew Gallatin break; 896e0501fd0SAndrew Gallatin default: 897b2fc195eSAndrew Gallatin device_printf(sc->dev, 8986d87a65dSAndrew Gallatin "mxge: command %d " 899b2fc195eSAndrew Gallatin "failed, result = %d\n", 900b2fc195eSAndrew Gallatin cmd, be32toh(response->result)); 901e0501fd0SAndrew Gallatin err = ENXIO; 902e0501fd0SAndrew Gallatin break; 903b2fc195eSAndrew Gallatin } 904e0501fd0SAndrew Gallatin if (err != EAGAIN) 905e0501fd0SAndrew Gallatin break; 906b2fc195eSAndrew Gallatin } 907e0501fd0SAndrew Gallatin if (err == EAGAIN) 9086d87a65dSAndrew Gallatin device_printf(sc->dev, "mxge: command %d timed out" 909b2fc195eSAndrew Gallatin "result = %d\n", 910b2fc195eSAndrew Gallatin cmd, be32toh(response->result)); 911e0501fd0SAndrew Gallatin mtx_unlock(&sc->cmd_mtx); 912e0501fd0SAndrew Gallatin return err; 913b2fc195eSAndrew Gallatin } 914b2fc195eSAndrew Gallatin 9154da0d523SAndrew Gallatin static int 9164da0d523SAndrew Gallatin mxge_adopt_running_firmware(mxge_softc_t *sc) 9174da0d523SAndrew Gallatin { 9184da0d523SAndrew Gallatin struct mcp_gen_header *hdr; 9194da0d523SAndrew Gallatin const size_t bytes = sizeof (struct mcp_gen_header); 9204da0d523SAndrew Gallatin size_t hdr_offset; 9214da0d523SAndrew Gallatin int status; 9224da0d523SAndrew Gallatin 9234da0d523SAndrew Gallatin /* find running firmware header */ 9244da0d523SAndrew Gallatin hdr_offset = htobe32(*(volatile uint32_t *) 9254da0d523SAndrew Gallatin (sc->sram + MCP_HEADER_PTR_OFFSET)); 9264da0d523SAndrew Gallatin 9274da0d523SAndrew Gallatin if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > sc->sram_size) { 9284da0d523SAndrew Gallatin device_printf(sc->dev, 9294da0d523SAndrew Gallatin "Running firmware has bad header offset (%d)\n", 9304da0d523SAndrew Gallatin (int)hdr_offset); 9314da0d523SAndrew Gallatin return EIO; 9324da0d523SAndrew Gallatin } 9334da0d523SAndrew Gallatin 9344da0d523SAndrew Gallatin /* copy header of running firmware from SRAM to host memory to 9354da0d523SAndrew Gallatin * validate firmware */ 9364da0d523SAndrew Gallatin hdr = malloc(bytes, M_DEVBUF, M_NOWAIT); 9374da0d523SAndrew Gallatin if (hdr == NULL) { 9384da0d523SAndrew Gallatin device_printf(sc->dev, "could not malloc firmware hdr\n"); 9394da0d523SAndrew Gallatin return ENOMEM; 9404da0d523SAndrew Gallatin } 9414da0d523SAndrew Gallatin bus_space_read_region_1(rman_get_bustag(sc->mem_res), 9424da0d523SAndrew Gallatin rman_get_bushandle(sc->mem_res), 9434da0d523SAndrew Gallatin hdr_offset, (char *)hdr, bytes); 9444da0d523SAndrew Gallatin status = mxge_validate_firmware(sc, hdr); 9454da0d523SAndrew Gallatin free(hdr, M_DEVBUF); 946b824b7d8SAndrew Gallatin 947b824b7d8SAndrew Gallatin /* 948b824b7d8SAndrew Gallatin * check to see if adopted firmware has bug where adopting 949b824b7d8SAndrew Gallatin * it will cause broadcasts to be filtered unless the NIC 950b824b7d8SAndrew Gallatin * is kept in ALLMULTI mode 951b824b7d8SAndrew Gallatin */ 952b824b7d8SAndrew Gallatin if (sc->fw_ver_major == 1 && sc->fw_ver_minor == 4 && 953b824b7d8SAndrew Gallatin sc->fw_ver_tiny >= 4 && sc->fw_ver_tiny <= 11) { 954b824b7d8SAndrew Gallatin sc->adopted_rx_filter_bug = 1; 955b824b7d8SAndrew Gallatin device_printf(sc->dev, "Adopting fw %d.%d.%d: " 956b824b7d8SAndrew Gallatin "working around rx filter bug\n", 957b824b7d8SAndrew Gallatin sc->fw_ver_major, sc->fw_ver_minor, 958b824b7d8SAndrew Gallatin sc->fw_ver_tiny); 959b824b7d8SAndrew Gallatin } 960b824b7d8SAndrew Gallatin 9614da0d523SAndrew Gallatin return status; 9624da0d523SAndrew Gallatin } 9634da0d523SAndrew Gallatin 964b2fc195eSAndrew Gallatin 965b2fc195eSAndrew Gallatin static int 9661e413cf9SAndrew Gallatin mxge_load_firmware(mxge_softc_t *sc, int adopt) 967b2fc195eSAndrew Gallatin { 968b2fc195eSAndrew Gallatin volatile uint32_t *confirm; 969b2fc195eSAndrew Gallatin volatile char *submit; 970b2fc195eSAndrew Gallatin char buf_bytes[72]; 971b2fc195eSAndrew Gallatin uint32_t *buf, size, dma_low, dma_high; 972b2fc195eSAndrew Gallatin int status, i; 973b2fc195eSAndrew Gallatin 974b2fc195eSAndrew Gallatin buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL); 975b2fc195eSAndrew Gallatin 976b2fc195eSAndrew Gallatin size = sc->sram_size; 9776d87a65dSAndrew Gallatin status = mxge_load_firmware_helper(sc, &size); 978b2fc195eSAndrew Gallatin if (status) { 9791e413cf9SAndrew Gallatin if (!adopt) 9801e413cf9SAndrew Gallatin return status; 9814da0d523SAndrew Gallatin /* Try to use the currently running firmware, if 9824da0d523SAndrew Gallatin it is new enough */ 9834da0d523SAndrew Gallatin status = mxge_adopt_running_firmware(sc); 9844da0d523SAndrew Gallatin if (status) { 9854da0d523SAndrew Gallatin device_printf(sc->dev, 9864da0d523SAndrew Gallatin "failed to adopt running firmware\n"); 987b2fc195eSAndrew Gallatin return status; 988b2fc195eSAndrew Gallatin } 9894da0d523SAndrew Gallatin device_printf(sc->dev, 9904da0d523SAndrew Gallatin "Successfully adopted running firmware\n"); 9911e413cf9SAndrew Gallatin if (sc->tx_boundary == 4096) { 9924da0d523SAndrew Gallatin device_printf(sc->dev, 9934da0d523SAndrew Gallatin "Using firmware currently running on NIC" 9944da0d523SAndrew Gallatin ". For optimal\n"); 9954da0d523SAndrew Gallatin device_printf(sc->dev, 9964da0d523SAndrew Gallatin "performance consider loading optimized " 9974da0d523SAndrew Gallatin "firmware\n"); 9984da0d523SAndrew Gallatin } 999d91b1b49SAndrew Gallatin sc->fw_name = mxge_fw_unaligned; 10001e413cf9SAndrew Gallatin sc->tx_boundary = 2048; 1001d91b1b49SAndrew Gallatin return 0; 10024da0d523SAndrew Gallatin } 1003b2fc195eSAndrew Gallatin /* clear confirmation addr */ 1004b2fc195eSAndrew Gallatin confirm = (volatile uint32_t *)sc->cmd; 1005b2fc195eSAndrew Gallatin *confirm = 0; 100673c7c83fSAndrew Gallatin wmb(); 1007b2fc195eSAndrew Gallatin /* send a reload command to the bootstrap MCP, and wait for the 1008b2fc195eSAndrew Gallatin response in the confirmation address. The firmware should 1009b2fc195eSAndrew Gallatin write a -1 there to indicate it is alive and well 1010b2fc195eSAndrew Gallatin */ 1011b2fc195eSAndrew Gallatin 10126d87a65dSAndrew Gallatin dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr); 10136d87a65dSAndrew Gallatin dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr); 1014b2fc195eSAndrew Gallatin 1015b2fc195eSAndrew Gallatin buf[0] = htobe32(dma_high); /* confirm addr MSW */ 1016b2fc195eSAndrew Gallatin buf[1] = htobe32(dma_low); /* confirm addr LSW */ 1017b2fc195eSAndrew Gallatin buf[2] = htobe32(0xffffffff); /* confirm data */ 1018b2fc195eSAndrew Gallatin 1019b2fc195eSAndrew Gallatin /* FIX: All newest firmware should un-protect the bottom of 1020b2fc195eSAndrew Gallatin the sram before handoff. However, the very first interfaces 1021b2fc195eSAndrew Gallatin do not. Therefore the handoff copy must skip the first 8 bytes 1022b2fc195eSAndrew Gallatin */ 1023b2fc195eSAndrew Gallatin /* where the code starts*/ 10246d87a65dSAndrew Gallatin buf[3] = htobe32(MXGE_FW_OFFSET + 8); 1025b2fc195eSAndrew Gallatin buf[4] = htobe32(size - 8); /* length of code */ 1026b2fc195eSAndrew Gallatin buf[5] = htobe32(8); /* where to copy to */ 1027b2fc195eSAndrew Gallatin buf[6] = htobe32(0); /* where to jump to */ 1028b2fc195eSAndrew Gallatin 10290fa7f681SAndrew Gallatin submit = (volatile char *)(sc->sram + MXGEFW_BOOT_HANDOFF); 10306d87a65dSAndrew Gallatin mxge_pio_copy(submit, buf, 64); 103173c7c83fSAndrew Gallatin wmb(); 1032b2fc195eSAndrew Gallatin DELAY(1000); 103373c7c83fSAndrew Gallatin wmb(); 1034b2fc195eSAndrew Gallatin i = 0; 1035b2fc195eSAndrew Gallatin while (*confirm != 0xffffffff && i < 20) { 1036b2fc195eSAndrew Gallatin DELAY(1000*10); 1037b2fc195eSAndrew Gallatin i++; 1038b2fc195eSAndrew Gallatin bus_dmamap_sync(sc->cmd_dma.dmat, 1039b2fc195eSAndrew Gallatin sc->cmd_dma.map, BUS_DMASYNC_POSTREAD); 1040b2fc195eSAndrew Gallatin } 1041b2fc195eSAndrew Gallatin if (*confirm != 0xffffffff) { 1042b2fc195eSAndrew Gallatin device_printf(sc->dev,"handoff failed (%p = 0x%x)", 1043b2fc195eSAndrew Gallatin confirm, *confirm); 1044b2fc195eSAndrew Gallatin 1045b2fc195eSAndrew Gallatin return ENXIO; 1046b2fc195eSAndrew Gallatin } 1047b2fc195eSAndrew Gallatin return 0; 1048b2fc195eSAndrew Gallatin } 1049b2fc195eSAndrew Gallatin 1050b2fc195eSAndrew Gallatin static int 10516d87a65dSAndrew Gallatin mxge_update_mac_address(mxge_softc_t *sc) 1052b2fc195eSAndrew Gallatin { 10536d87a65dSAndrew Gallatin mxge_cmd_t cmd; 1054b2fc195eSAndrew Gallatin uint8_t *addr = sc->mac_addr; 1055b2fc195eSAndrew Gallatin int status; 1056b2fc195eSAndrew Gallatin 1057b2fc195eSAndrew Gallatin 1058b2fc195eSAndrew Gallatin cmd.data0 = ((addr[0] << 24) | (addr[1] << 16) 1059b2fc195eSAndrew Gallatin | (addr[2] << 8) | addr[3]); 1060b2fc195eSAndrew Gallatin 1061b2fc195eSAndrew Gallatin cmd.data1 = ((addr[4] << 8) | (addr[5])); 1062b2fc195eSAndrew Gallatin 10635e7d8541SAndrew Gallatin status = mxge_send_cmd(sc, MXGEFW_SET_MAC_ADDRESS, &cmd); 1064b2fc195eSAndrew Gallatin return status; 1065b2fc195eSAndrew Gallatin } 1066b2fc195eSAndrew Gallatin 1067b2fc195eSAndrew Gallatin static int 10686d87a65dSAndrew Gallatin mxge_change_pause(mxge_softc_t *sc, int pause) 1069b2fc195eSAndrew Gallatin { 10706d87a65dSAndrew Gallatin mxge_cmd_t cmd; 1071b2fc195eSAndrew Gallatin int status; 1072b2fc195eSAndrew Gallatin 1073b2fc195eSAndrew Gallatin if (pause) 10745e7d8541SAndrew Gallatin status = mxge_send_cmd(sc, MXGEFW_ENABLE_FLOW_CONTROL, 1075b2fc195eSAndrew Gallatin &cmd); 1076b2fc195eSAndrew Gallatin else 10775e7d8541SAndrew Gallatin status = mxge_send_cmd(sc, MXGEFW_DISABLE_FLOW_CONTROL, 1078b2fc195eSAndrew Gallatin &cmd); 1079b2fc195eSAndrew Gallatin 1080b2fc195eSAndrew Gallatin if (status) { 1081b2fc195eSAndrew Gallatin device_printf(sc->dev, "Failed to set flow control mode\n"); 1082b2fc195eSAndrew Gallatin return ENXIO; 1083b2fc195eSAndrew Gallatin } 1084b2fc195eSAndrew Gallatin sc->pause = pause; 1085b2fc195eSAndrew Gallatin return 0; 1086b2fc195eSAndrew Gallatin } 1087b2fc195eSAndrew Gallatin 1088b2fc195eSAndrew Gallatin static void 10896d87a65dSAndrew Gallatin mxge_change_promisc(mxge_softc_t *sc, int promisc) 1090b2fc195eSAndrew Gallatin { 10916d87a65dSAndrew Gallatin mxge_cmd_t cmd; 1092b2fc195eSAndrew Gallatin int status; 1093b2fc195eSAndrew Gallatin 10941e413cf9SAndrew Gallatin if (mxge_always_promisc) 10951e413cf9SAndrew Gallatin promisc = 1; 10961e413cf9SAndrew Gallatin 1097b2fc195eSAndrew Gallatin if (promisc) 10985e7d8541SAndrew Gallatin status = mxge_send_cmd(sc, MXGEFW_ENABLE_PROMISC, 1099b2fc195eSAndrew Gallatin &cmd); 1100b2fc195eSAndrew Gallatin else 11015e7d8541SAndrew Gallatin status = mxge_send_cmd(sc, MXGEFW_DISABLE_PROMISC, 1102b2fc195eSAndrew Gallatin &cmd); 1103b2fc195eSAndrew Gallatin 1104b2fc195eSAndrew Gallatin if (status) { 1105b2fc195eSAndrew Gallatin device_printf(sc->dev, "Failed to set promisc mode\n"); 1106b2fc195eSAndrew Gallatin } 1107b2fc195eSAndrew Gallatin } 1108b2fc195eSAndrew Gallatin 11090fa7f681SAndrew Gallatin static void 11100fa7f681SAndrew Gallatin mxge_set_multicast_list(mxge_softc_t *sc) 11110fa7f681SAndrew Gallatin { 11120fa7f681SAndrew Gallatin mxge_cmd_t cmd; 11130fa7f681SAndrew Gallatin struct ifmultiaddr *ifma; 11140fa7f681SAndrew Gallatin struct ifnet *ifp = sc->ifp; 11150fa7f681SAndrew Gallatin int err; 11160fa7f681SAndrew Gallatin 11170fa7f681SAndrew Gallatin /* This firmware is known to not support multicast */ 11180fa7f681SAndrew Gallatin if (!sc->fw_multicast_support) 11190fa7f681SAndrew Gallatin return; 11200fa7f681SAndrew Gallatin 11210fa7f681SAndrew Gallatin /* Disable multicast filtering while we play with the lists*/ 11220fa7f681SAndrew Gallatin err = mxge_send_cmd(sc, MXGEFW_ENABLE_ALLMULTI, &cmd); 11230fa7f681SAndrew Gallatin if (err != 0) { 11240fa7f681SAndrew Gallatin device_printf(sc->dev, "Failed MXGEFW_ENABLE_ALLMULTI," 11250fa7f681SAndrew Gallatin " error status: %d\n", err); 11260fa7f681SAndrew Gallatin return; 11270fa7f681SAndrew Gallatin } 11280fa7f681SAndrew Gallatin 1129b824b7d8SAndrew Gallatin if (sc->adopted_rx_filter_bug) 1130b824b7d8SAndrew Gallatin return; 11310fa7f681SAndrew Gallatin 11320fa7f681SAndrew Gallatin if (ifp->if_flags & IFF_ALLMULTI) 11330fa7f681SAndrew Gallatin /* request to disable multicast filtering, so quit here */ 11340fa7f681SAndrew Gallatin return; 11350fa7f681SAndrew Gallatin 11360fa7f681SAndrew Gallatin /* Flush all the filters */ 11370fa7f681SAndrew Gallatin 11380fa7f681SAndrew Gallatin err = mxge_send_cmd(sc, MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, &cmd); 11390fa7f681SAndrew Gallatin if (err != 0) { 11400fa7f681SAndrew Gallatin device_printf(sc->dev, 11410fa7f681SAndrew Gallatin "Failed MXGEFW_LEAVE_ALL_MULTICAST_GROUPS" 11420fa7f681SAndrew Gallatin ", error status: %d\n", err); 11430fa7f681SAndrew Gallatin return; 11440fa7f681SAndrew Gallatin } 11450fa7f681SAndrew Gallatin 11460fa7f681SAndrew Gallatin /* Walk the multicast list, and add each address */ 11470fa7f681SAndrew Gallatin 1148eb956cd0SRobert Watson if_maddr_rlock(ifp); 11490fa7f681SAndrew Gallatin TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 11500fa7f681SAndrew Gallatin if (ifma->ifma_addr->sa_family != AF_LINK) 11510fa7f681SAndrew Gallatin continue; 11520fa7f681SAndrew Gallatin bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 11530fa7f681SAndrew Gallatin &cmd.data0, 4); 11540fa7f681SAndrew Gallatin bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr) + 4, 11550fa7f681SAndrew Gallatin &cmd.data1, 2); 11560fa7f681SAndrew Gallatin cmd.data0 = htonl(cmd.data0); 11570fa7f681SAndrew Gallatin cmd.data1 = htonl(cmd.data1); 11580fa7f681SAndrew Gallatin err = mxge_send_cmd(sc, MXGEFW_JOIN_MULTICAST_GROUP, &cmd); 11590fa7f681SAndrew Gallatin if (err != 0) { 11600fa7f681SAndrew Gallatin device_printf(sc->dev, "Failed " 11610fa7f681SAndrew Gallatin "MXGEFW_JOIN_MULTICAST_GROUP, error status:" 11620fa7f681SAndrew Gallatin "%d\t", err); 11630fa7f681SAndrew Gallatin /* abort, leaving multicast filtering off */ 1164eb956cd0SRobert Watson if_maddr_runlock(ifp); 11650fa7f681SAndrew Gallatin return; 11660fa7f681SAndrew Gallatin } 11670fa7f681SAndrew Gallatin } 1168eb956cd0SRobert Watson if_maddr_runlock(ifp); 11690fa7f681SAndrew Gallatin /* Enable multicast filtering */ 11700fa7f681SAndrew Gallatin err = mxge_send_cmd(sc, MXGEFW_DISABLE_ALLMULTI, &cmd); 11710fa7f681SAndrew Gallatin if (err != 0) { 11720fa7f681SAndrew Gallatin device_printf(sc->dev, "Failed MXGEFW_DISABLE_ALLMULTI" 11730fa7f681SAndrew Gallatin ", error status: %d\n", err); 11740fa7f681SAndrew Gallatin } 11750fa7f681SAndrew Gallatin } 11760fa7f681SAndrew Gallatin 1177b2fc195eSAndrew Gallatin static int 1178053e637fSAndrew Gallatin mxge_max_mtu(mxge_softc_t *sc) 1179053e637fSAndrew Gallatin { 1180053e637fSAndrew Gallatin mxge_cmd_t cmd; 1181053e637fSAndrew Gallatin int status; 1182053e637fSAndrew Gallatin 1183c792928fSAndrew Gallatin if (MJUMPAGESIZE - MXGEFW_PAD > MXGEFW_MAX_MTU) 1184c792928fSAndrew Gallatin return MXGEFW_MAX_MTU - MXGEFW_PAD; 1185053e637fSAndrew Gallatin 1186053e637fSAndrew Gallatin /* try to set nbufs to see if it we can 1187053e637fSAndrew Gallatin use virtually contiguous jumbos */ 1188053e637fSAndrew Gallatin cmd.data0 = 0; 1189053e637fSAndrew Gallatin status = mxge_send_cmd(sc, MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS, 1190053e637fSAndrew Gallatin &cmd); 1191053e637fSAndrew Gallatin if (status == 0) 1192c792928fSAndrew Gallatin return MXGEFW_MAX_MTU - MXGEFW_PAD; 1193053e637fSAndrew Gallatin 1194053e637fSAndrew Gallatin /* otherwise, we're limited to MJUMPAGESIZE */ 1195053e637fSAndrew Gallatin return MJUMPAGESIZE - MXGEFW_PAD; 1196053e637fSAndrew Gallatin } 1197053e637fSAndrew Gallatin 1198053e637fSAndrew Gallatin static int 1199adae7080SAndrew Gallatin mxge_reset(mxge_softc_t *sc, int interrupts_setup) 1200b2fc195eSAndrew Gallatin { 12011e413cf9SAndrew Gallatin struct mxge_slice_state *ss; 12021e413cf9SAndrew Gallatin mxge_rx_done_t *rx_done; 12031e413cf9SAndrew Gallatin volatile uint32_t *irq_claim; 12046d87a65dSAndrew Gallatin mxge_cmd_t cmd; 12051e413cf9SAndrew Gallatin int slice, status; 1206b2fc195eSAndrew Gallatin 1207b2fc195eSAndrew Gallatin /* try to send a reset command to the card to see if it 1208b2fc195eSAndrew Gallatin is alive */ 1209b2fc195eSAndrew Gallatin memset(&cmd, 0, sizeof (cmd)); 12105e7d8541SAndrew Gallatin status = mxge_send_cmd(sc, MXGEFW_CMD_RESET, &cmd); 1211b2fc195eSAndrew Gallatin if (status != 0) { 1212b2fc195eSAndrew Gallatin device_printf(sc->dev, "failed reset\n"); 1213b2fc195eSAndrew Gallatin return ENXIO; 1214b2fc195eSAndrew Gallatin } 1215b2fc195eSAndrew Gallatin 1216091feecdSAndrew Gallatin mxge_dummy_rdma(sc, 1); 1217091feecdSAndrew Gallatin 12181e413cf9SAndrew Gallatin 12191e413cf9SAndrew Gallatin /* set the intrq size */ 12201e413cf9SAndrew Gallatin cmd.data0 = sc->rx_ring_size; 12211e413cf9SAndrew Gallatin status = mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd); 12221e413cf9SAndrew Gallatin 12231e413cf9SAndrew Gallatin /* 12241e413cf9SAndrew Gallatin * Even though we already know how many slices are supported 12251e413cf9SAndrew Gallatin * via mxge_slice_probe(), MXGEFW_CMD_GET_MAX_RSS_QUEUES 12261e413cf9SAndrew Gallatin * has magic side effects, and must be called after a reset. 12271e413cf9SAndrew Gallatin * It must be called prior to calling any RSS related cmds, 12281e413cf9SAndrew Gallatin * including assigning an interrupt queue for anything but 12291e413cf9SAndrew Gallatin * slice 0. It must also be called *after* 12301e413cf9SAndrew Gallatin * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by 12311e413cf9SAndrew Gallatin * the firmware to compute offsets. 12321e413cf9SAndrew Gallatin */ 12331e413cf9SAndrew Gallatin 12341e413cf9SAndrew Gallatin if (sc->num_slices > 1) { 12351e413cf9SAndrew Gallatin /* ask the maximum number of slices it supports */ 12361e413cf9SAndrew Gallatin status = mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_RSS_QUEUES, 12371e413cf9SAndrew Gallatin &cmd); 12381e413cf9SAndrew Gallatin if (status != 0) { 12391e413cf9SAndrew Gallatin device_printf(sc->dev, 12401e413cf9SAndrew Gallatin "failed to get number of slices\n"); 12411e413cf9SAndrew Gallatin return status; 12421e413cf9SAndrew Gallatin } 12431e413cf9SAndrew Gallatin /* 12441e413cf9SAndrew Gallatin * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior 12451e413cf9SAndrew Gallatin * to setting up the interrupt queue DMA 12461e413cf9SAndrew Gallatin */ 12471e413cf9SAndrew Gallatin cmd.data0 = sc->num_slices; 12481e413cf9SAndrew Gallatin cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 1249c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING 1250c6cb3e3fSAndrew Gallatin cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 1251c6cb3e3fSAndrew Gallatin #endif 12521e413cf9SAndrew Gallatin status = mxge_send_cmd(sc, MXGEFW_CMD_ENABLE_RSS_QUEUES, 12531e413cf9SAndrew Gallatin &cmd); 12541e413cf9SAndrew Gallatin if (status != 0) { 12551e413cf9SAndrew Gallatin device_printf(sc->dev, 12561e413cf9SAndrew Gallatin "failed to set number of slices\n"); 12571e413cf9SAndrew Gallatin return status; 12581e413cf9SAndrew Gallatin } 12591e413cf9SAndrew Gallatin } 12601e413cf9SAndrew Gallatin 12611e413cf9SAndrew Gallatin 1262adae7080SAndrew Gallatin if (interrupts_setup) { 1263b2fc195eSAndrew Gallatin /* Now exchange information about interrupts */ 12641e413cf9SAndrew Gallatin for (slice = 0; slice < sc->num_slices; slice++) { 12651e413cf9SAndrew Gallatin rx_done = &sc->ss[slice].rx_done; 12661e413cf9SAndrew Gallatin memset(rx_done->entry, 0, sc->rx_ring_size); 12671e413cf9SAndrew Gallatin cmd.data0 = MXGE_LOWPART_TO_U32(rx_done->dma.bus_addr); 12681e413cf9SAndrew Gallatin cmd.data1 = MXGE_HIGHPART_TO_U32(rx_done->dma.bus_addr); 12691e413cf9SAndrew Gallatin cmd.data2 = slice; 12701e413cf9SAndrew Gallatin status |= mxge_send_cmd(sc, 12711e413cf9SAndrew Gallatin MXGEFW_CMD_SET_INTRQ_DMA, 12721e413cf9SAndrew Gallatin &cmd); 12731e413cf9SAndrew Gallatin } 1274adae7080SAndrew Gallatin } 1275b2fc195eSAndrew Gallatin 12766d87a65dSAndrew Gallatin status |= mxge_send_cmd(sc, 12775e7d8541SAndrew Gallatin MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd); 12785e7d8541SAndrew Gallatin 12795e7d8541SAndrew Gallatin 12805e7d8541SAndrew Gallatin sc->intr_coal_delay_ptr = (volatile uint32_t *)(sc->sram + cmd.data0); 12815e7d8541SAndrew Gallatin 12825e7d8541SAndrew Gallatin status |= mxge_send_cmd(sc, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd); 12831e413cf9SAndrew Gallatin irq_claim = (volatile uint32_t *)(sc->sram + cmd.data0); 12845e7d8541SAndrew Gallatin 12855e7d8541SAndrew Gallatin 12865e7d8541SAndrew Gallatin status |= mxge_send_cmd(sc, MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET, 12876d87a65dSAndrew Gallatin &cmd); 12885e7d8541SAndrew Gallatin sc->irq_deassert = (volatile uint32_t *)(sc->sram + cmd.data0); 1289b2fc195eSAndrew Gallatin if (status != 0) { 1290b2fc195eSAndrew Gallatin device_printf(sc->dev, "failed set interrupt parameters\n"); 1291b2fc195eSAndrew Gallatin return status; 1292b2fc195eSAndrew Gallatin } 1293b2fc195eSAndrew Gallatin 12945e7d8541SAndrew Gallatin 12955e7d8541SAndrew Gallatin *sc->intr_coal_delay_ptr = htobe32(sc->intr_coal_delay); 12965e7d8541SAndrew Gallatin 12975e7d8541SAndrew Gallatin 12985e7d8541SAndrew Gallatin /* run a DMA benchmark */ 12998fe615baSAndrew Gallatin (void) mxge_dma_test(sc, MXGEFW_DMA_TEST); 13005e7d8541SAndrew Gallatin 13011e413cf9SAndrew Gallatin for (slice = 0; slice < sc->num_slices; slice++) { 13021e413cf9SAndrew Gallatin ss = &sc->ss[slice]; 13031e413cf9SAndrew Gallatin 13041e413cf9SAndrew Gallatin ss->irq_claim = irq_claim + (2 * slice); 1305b2fc195eSAndrew Gallatin /* reset mcp/driver shared state back to 0 */ 13061e413cf9SAndrew Gallatin ss->rx_done.idx = 0; 13071e413cf9SAndrew Gallatin ss->rx_done.cnt = 0; 13081e413cf9SAndrew Gallatin ss->tx.req = 0; 13091e413cf9SAndrew Gallatin ss->tx.done = 0; 13101e413cf9SAndrew Gallatin ss->tx.pkt_done = 0; 1311c6cb3e3fSAndrew Gallatin ss->tx.queue_active = 0; 1312c6cb3e3fSAndrew Gallatin ss->tx.activate = 0; 1313c6cb3e3fSAndrew Gallatin ss->tx.deactivate = 0; 13141e413cf9SAndrew Gallatin ss->tx.wake = 0; 13151e413cf9SAndrew Gallatin ss->tx.defrag = 0; 13161e413cf9SAndrew Gallatin ss->tx.stall = 0; 13171e413cf9SAndrew Gallatin ss->rx_big.cnt = 0; 13181e413cf9SAndrew Gallatin ss->rx_small.cnt = 0; 131926dd49c6SAndrew Gallatin ss->lc.lro_bad_csum = 0; 132026dd49c6SAndrew Gallatin ss->lc.lro_queued = 0; 132126dd49c6SAndrew Gallatin ss->lc.lro_flushed = 0; 13221e413cf9SAndrew Gallatin if (ss->fw_stats != NULL) { 1323a393336bSAndrew Gallatin bzero(ss->fw_stats, sizeof *ss->fw_stats); 13241e413cf9SAndrew Gallatin } 13251e413cf9SAndrew Gallatin } 1326b2fc195eSAndrew Gallatin sc->rdma_tags_available = 15; 13276d87a65dSAndrew Gallatin status = mxge_update_mac_address(sc); 1328bb8ddc66SAndrew Gallatin mxge_change_promisc(sc, sc->ifp->if_flags & IFF_PROMISC); 13296d87a65dSAndrew Gallatin mxge_change_pause(sc, sc->pause); 13300fa7f681SAndrew Gallatin mxge_set_multicast_list(sc); 133165c69066SAndrew Gallatin if (sc->throttle) { 133265c69066SAndrew Gallatin cmd.data0 = sc->throttle; 133365c69066SAndrew Gallatin if (mxge_send_cmd(sc, MXGEFW_CMD_SET_THROTTLE_FACTOR, 133465c69066SAndrew Gallatin &cmd)) { 133565c69066SAndrew Gallatin device_printf(sc->dev, 133665c69066SAndrew Gallatin "can't enable throttle\n"); 133765c69066SAndrew Gallatin } 133865c69066SAndrew Gallatin } 1339b2fc195eSAndrew Gallatin return status; 1340b2fc195eSAndrew Gallatin } 1341b2fc195eSAndrew Gallatin 1342b2fc195eSAndrew Gallatin static int 134365c69066SAndrew Gallatin mxge_change_throttle(SYSCTL_HANDLER_ARGS) 134465c69066SAndrew Gallatin { 134565c69066SAndrew Gallatin mxge_cmd_t cmd; 134665c69066SAndrew Gallatin mxge_softc_t *sc; 134765c69066SAndrew Gallatin int err; 134865c69066SAndrew Gallatin unsigned int throttle; 134965c69066SAndrew Gallatin 135065c69066SAndrew Gallatin sc = arg1; 135165c69066SAndrew Gallatin throttle = sc->throttle; 135265c69066SAndrew Gallatin err = sysctl_handle_int(oidp, &throttle, arg2, req); 135365c69066SAndrew Gallatin if (err != 0) { 135465c69066SAndrew Gallatin return err; 135565c69066SAndrew Gallatin } 135665c69066SAndrew Gallatin 135765c69066SAndrew Gallatin if (throttle == sc->throttle) 135865c69066SAndrew Gallatin return 0; 135965c69066SAndrew Gallatin 136065c69066SAndrew Gallatin if (throttle < MXGE_MIN_THROTTLE || throttle > MXGE_MAX_THROTTLE) 136165c69066SAndrew Gallatin return EINVAL; 136265c69066SAndrew Gallatin 136365c69066SAndrew Gallatin mtx_lock(&sc->driver_mtx); 136465c69066SAndrew Gallatin cmd.data0 = throttle; 136565c69066SAndrew Gallatin err = mxge_send_cmd(sc, MXGEFW_CMD_SET_THROTTLE_FACTOR, &cmd); 136665c69066SAndrew Gallatin if (err == 0) 136765c69066SAndrew Gallatin sc->throttle = throttle; 136865c69066SAndrew Gallatin mtx_unlock(&sc->driver_mtx); 136965c69066SAndrew Gallatin return err; 137065c69066SAndrew Gallatin } 137165c69066SAndrew Gallatin 137265c69066SAndrew Gallatin static int 13736d87a65dSAndrew Gallatin mxge_change_intr_coal(SYSCTL_HANDLER_ARGS) 1374b2fc195eSAndrew Gallatin { 13756d87a65dSAndrew Gallatin mxge_softc_t *sc; 1376b2fc195eSAndrew Gallatin unsigned int intr_coal_delay; 1377b2fc195eSAndrew Gallatin int err; 1378b2fc195eSAndrew Gallatin 1379b2fc195eSAndrew Gallatin sc = arg1; 1380b2fc195eSAndrew Gallatin intr_coal_delay = sc->intr_coal_delay; 1381b2fc195eSAndrew Gallatin err = sysctl_handle_int(oidp, &intr_coal_delay, arg2, req); 1382b2fc195eSAndrew Gallatin if (err != 0) { 1383b2fc195eSAndrew Gallatin return err; 1384b2fc195eSAndrew Gallatin } 1385b2fc195eSAndrew Gallatin if (intr_coal_delay == sc->intr_coal_delay) 1386b2fc195eSAndrew Gallatin return 0; 1387b2fc195eSAndrew Gallatin 1388b2fc195eSAndrew Gallatin if (intr_coal_delay == 0 || intr_coal_delay > 1000*1000) 1389b2fc195eSAndrew Gallatin return EINVAL; 1390b2fc195eSAndrew Gallatin 1391a98d6cd7SAndrew Gallatin mtx_lock(&sc->driver_mtx); 13925e7d8541SAndrew Gallatin *sc->intr_coal_delay_ptr = htobe32(intr_coal_delay); 1393b2fc195eSAndrew Gallatin sc->intr_coal_delay = intr_coal_delay; 13945e7d8541SAndrew Gallatin 1395a98d6cd7SAndrew Gallatin mtx_unlock(&sc->driver_mtx); 1396b2fc195eSAndrew Gallatin return err; 1397b2fc195eSAndrew Gallatin } 1398b2fc195eSAndrew Gallatin 1399b2fc195eSAndrew Gallatin static int 14006d87a65dSAndrew Gallatin mxge_change_flow_control(SYSCTL_HANDLER_ARGS) 1401b2fc195eSAndrew Gallatin { 14026d87a65dSAndrew Gallatin mxge_softc_t *sc; 1403b2fc195eSAndrew Gallatin unsigned int enabled; 1404b2fc195eSAndrew Gallatin int err; 1405b2fc195eSAndrew Gallatin 1406b2fc195eSAndrew Gallatin sc = arg1; 1407b2fc195eSAndrew Gallatin enabled = sc->pause; 1408b2fc195eSAndrew Gallatin err = sysctl_handle_int(oidp, &enabled, arg2, req); 1409b2fc195eSAndrew Gallatin if (err != 0) { 1410b2fc195eSAndrew Gallatin return err; 1411b2fc195eSAndrew Gallatin } 1412b2fc195eSAndrew Gallatin if (enabled == sc->pause) 1413b2fc195eSAndrew Gallatin return 0; 1414b2fc195eSAndrew Gallatin 1415a98d6cd7SAndrew Gallatin mtx_lock(&sc->driver_mtx); 14166d87a65dSAndrew Gallatin err = mxge_change_pause(sc, enabled); 1417a98d6cd7SAndrew Gallatin mtx_unlock(&sc->driver_mtx); 1418b2fc195eSAndrew Gallatin return err; 1419b2fc195eSAndrew Gallatin } 1420b2fc195eSAndrew Gallatin 1421b2fc195eSAndrew Gallatin static int 14226d87a65dSAndrew Gallatin mxge_handle_be32(SYSCTL_HANDLER_ARGS) 1423b2fc195eSAndrew Gallatin { 1424b2fc195eSAndrew Gallatin int err; 1425b2fc195eSAndrew Gallatin 1426b2fc195eSAndrew Gallatin if (arg1 == NULL) 1427b2fc195eSAndrew Gallatin return EFAULT; 1428b2fc195eSAndrew Gallatin arg2 = be32toh(*(int *)arg1); 1429b2fc195eSAndrew Gallatin arg1 = NULL; 1430b2fc195eSAndrew Gallatin err = sysctl_handle_int(oidp, arg1, arg2, req); 1431b2fc195eSAndrew Gallatin 1432b2fc195eSAndrew Gallatin return err; 1433b2fc195eSAndrew Gallatin } 1434b2fc195eSAndrew Gallatin 1435b2fc195eSAndrew Gallatin static void 14361e413cf9SAndrew Gallatin mxge_rem_sysctls(mxge_softc_t *sc) 14371e413cf9SAndrew Gallatin { 14381e413cf9SAndrew Gallatin struct mxge_slice_state *ss; 14391e413cf9SAndrew Gallatin int slice; 14401e413cf9SAndrew Gallatin 14411e413cf9SAndrew Gallatin if (sc->slice_sysctl_tree == NULL) 14421e413cf9SAndrew Gallatin return; 14431e413cf9SAndrew Gallatin 14441e413cf9SAndrew Gallatin for (slice = 0; slice < sc->num_slices; slice++) { 14451e413cf9SAndrew Gallatin ss = &sc->ss[slice]; 14461e413cf9SAndrew Gallatin if (ss == NULL || ss->sysctl_tree == NULL) 14471e413cf9SAndrew Gallatin continue; 14481e413cf9SAndrew Gallatin sysctl_ctx_free(&ss->sysctl_ctx); 14491e413cf9SAndrew Gallatin ss->sysctl_tree = NULL; 14501e413cf9SAndrew Gallatin } 14511e413cf9SAndrew Gallatin sysctl_ctx_free(&sc->slice_sysctl_ctx); 14521e413cf9SAndrew Gallatin sc->slice_sysctl_tree = NULL; 14531e413cf9SAndrew Gallatin } 14541e413cf9SAndrew Gallatin 14551e413cf9SAndrew Gallatin static void 14566d87a65dSAndrew Gallatin mxge_add_sysctls(mxge_softc_t *sc) 1457b2fc195eSAndrew Gallatin { 1458b2fc195eSAndrew Gallatin struct sysctl_ctx_list *ctx; 1459b2fc195eSAndrew Gallatin struct sysctl_oid_list *children; 14605e7d8541SAndrew Gallatin mcp_irq_data_t *fw; 14611e413cf9SAndrew Gallatin struct mxge_slice_state *ss; 14621e413cf9SAndrew Gallatin int slice; 14631e413cf9SAndrew Gallatin char slice_num[8]; 1464b2fc195eSAndrew Gallatin 1465b2fc195eSAndrew Gallatin ctx = device_get_sysctl_ctx(sc->dev); 1466b2fc195eSAndrew Gallatin children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 14671e413cf9SAndrew Gallatin fw = sc->ss[0].fw_stats; 1468b2fc195eSAndrew Gallatin 14695e7d8541SAndrew Gallatin /* random information */ 14705e7d8541SAndrew Gallatin SYSCTL_ADD_STRING(ctx, children, OID_AUTO, 14715e7d8541SAndrew Gallatin "firmware_version", 1472f0188618SHans Petter Selasky CTLFLAG_RD, sc->fw_version, 14735e7d8541SAndrew Gallatin 0, "firmware version"); 14745e7d8541SAndrew Gallatin SYSCTL_ADD_STRING(ctx, children, OID_AUTO, 14755e7d8541SAndrew Gallatin "serial_number", 1476f0188618SHans Petter Selasky CTLFLAG_RD, sc->serial_number_string, 14775e7d8541SAndrew Gallatin 0, "serial number"); 14785e7d8541SAndrew Gallatin SYSCTL_ADD_STRING(ctx, children, OID_AUTO, 14795e7d8541SAndrew Gallatin "product_code", 1480f0188618SHans Petter Selasky CTLFLAG_RD, sc->product_code_string, 14815e7d8541SAndrew Gallatin 0, "product_code"); 14825e7d8541SAndrew Gallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1483d91b1b49SAndrew Gallatin "pcie_link_width", 1484d91b1b49SAndrew Gallatin CTLFLAG_RD, &sc->link_width, 1485d91b1b49SAndrew Gallatin 0, "tx_boundary"); 1486d91b1b49SAndrew Gallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 14875e7d8541SAndrew Gallatin "tx_boundary", 14881e413cf9SAndrew Gallatin CTLFLAG_RD, &sc->tx_boundary, 14895e7d8541SAndrew Gallatin 0, "tx_boundary"); 14905e7d8541SAndrew Gallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1491091feecdSAndrew Gallatin "write_combine", 1492091feecdSAndrew Gallatin CTLFLAG_RD, &sc->wc, 1493091feecdSAndrew Gallatin 0, "write combining PIO?"); 1494091feecdSAndrew Gallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 14955e7d8541SAndrew Gallatin "read_dma_MBs", 14965e7d8541SAndrew Gallatin CTLFLAG_RD, &sc->read_dma, 14975e7d8541SAndrew Gallatin 0, "DMA Read speed in MB/s"); 14985e7d8541SAndrew Gallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 14995e7d8541SAndrew Gallatin "write_dma_MBs", 15005e7d8541SAndrew Gallatin CTLFLAG_RD, &sc->write_dma, 15015e7d8541SAndrew Gallatin 0, "DMA Write speed in MB/s"); 15025e7d8541SAndrew Gallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 15035e7d8541SAndrew Gallatin "read_write_dma_MBs", 15045e7d8541SAndrew Gallatin CTLFLAG_RD, &sc->read_write_dma, 15055e7d8541SAndrew Gallatin 0, "DMA concurrent Read/Write speed in MB/s"); 1506a393336bSAndrew Gallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1507a393336bSAndrew Gallatin "watchdog_resets", 1508a393336bSAndrew Gallatin CTLFLAG_RD, &sc->watchdog_resets, 1509a393336bSAndrew Gallatin 0, "Number of times NIC was reset"); 15105e7d8541SAndrew Gallatin 15115e7d8541SAndrew Gallatin 15125e7d8541SAndrew Gallatin /* performance related tunables */ 1513b2fc195eSAndrew Gallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1514b2fc195eSAndrew Gallatin "intr_coal_delay", 1515b2fc195eSAndrew Gallatin CTLTYPE_INT|CTLFLAG_RW, sc, 15166d87a65dSAndrew Gallatin 0, mxge_change_intr_coal, 1517b2fc195eSAndrew Gallatin "I", "interrupt coalescing delay in usecs"); 1518b2fc195eSAndrew Gallatin 1519b2fc195eSAndrew Gallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 152065c69066SAndrew Gallatin "throttle", 152165c69066SAndrew Gallatin CTLTYPE_INT|CTLFLAG_RW, sc, 152265c69066SAndrew Gallatin 0, mxge_change_throttle, 152365c69066SAndrew Gallatin "I", "transmit throttling"); 152465c69066SAndrew Gallatin 152565c69066SAndrew Gallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1526b2fc195eSAndrew Gallatin "flow_control_enabled", 1527b2fc195eSAndrew Gallatin CTLTYPE_INT|CTLFLAG_RW, sc, 15286d87a65dSAndrew Gallatin 0, mxge_change_flow_control, 1529b2fc195eSAndrew Gallatin "I", "interrupt coalescing delay in usecs"); 1530b2fc195eSAndrew Gallatin 1531b2fc195eSAndrew Gallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 15325e7d8541SAndrew Gallatin "deassert_wait", 15335e7d8541SAndrew Gallatin CTLFLAG_RW, &mxge_deassert_wait, 15345e7d8541SAndrew Gallatin 0, "Wait for IRQ line to go low in ihandler"); 1535b2fc195eSAndrew Gallatin 1536b2fc195eSAndrew Gallatin /* stats block from firmware is in network byte order. 1537b2fc195eSAndrew Gallatin Need to swap it */ 1538b2fc195eSAndrew Gallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1539b2fc195eSAndrew Gallatin "link_up", 1540b2fc195eSAndrew Gallatin CTLTYPE_INT|CTLFLAG_RD, &fw->link_up, 15416d87a65dSAndrew Gallatin 0, mxge_handle_be32, 1542b2fc195eSAndrew Gallatin "I", "link up"); 1543b2fc195eSAndrew Gallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1544b2fc195eSAndrew Gallatin "rdma_tags_available", 1545b2fc195eSAndrew Gallatin CTLTYPE_INT|CTLFLAG_RD, &fw->rdma_tags_available, 15466d87a65dSAndrew Gallatin 0, mxge_handle_be32, 1547b2fc195eSAndrew Gallatin "I", "rdma_tags_available"); 1548b2fc195eSAndrew Gallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1549adae7080SAndrew Gallatin "dropped_bad_crc32", 1550adae7080SAndrew Gallatin CTLTYPE_INT|CTLFLAG_RD, 1551adae7080SAndrew Gallatin &fw->dropped_bad_crc32, 15526d87a65dSAndrew Gallatin 0, mxge_handle_be32, 1553adae7080SAndrew Gallatin "I", "dropped_bad_crc32"); 1554adae7080SAndrew Gallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1555adae7080SAndrew Gallatin "dropped_bad_phy", 1556adae7080SAndrew Gallatin CTLTYPE_INT|CTLFLAG_RD, 1557adae7080SAndrew Gallatin &fw->dropped_bad_phy, 1558adae7080SAndrew Gallatin 0, mxge_handle_be32, 1559adae7080SAndrew Gallatin "I", "dropped_bad_phy"); 1560b2fc195eSAndrew Gallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1561b2fc195eSAndrew Gallatin "dropped_link_error_or_filtered", 1562b2fc195eSAndrew Gallatin CTLTYPE_INT|CTLFLAG_RD, 1563b2fc195eSAndrew Gallatin &fw->dropped_link_error_or_filtered, 15646d87a65dSAndrew Gallatin 0, mxge_handle_be32, 1565b2fc195eSAndrew Gallatin "I", "dropped_link_error_or_filtered"); 1566b2fc195eSAndrew Gallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1567adae7080SAndrew Gallatin "dropped_link_overflow", 1568adae7080SAndrew Gallatin CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_link_overflow, 1569adae7080SAndrew Gallatin 0, mxge_handle_be32, 1570adae7080SAndrew Gallatin "I", "dropped_link_overflow"); 1571adae7080SAndrew Gallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 15720fa7f681SAndrew Gallatin "dropped_multicast_filtered", 15730fa7f681SAndrew Gallatin CTLTYPE_INT|CTLFLAG_RD, 15740fa7f681SAndrew Gallatin &fw->dropped_multicast_filtered, 15750fa7f681SAndrew Gallatin 0, mxge_handle_be32, 15760fa7f681SAndrew Gallatin "I", "dropped_multicast_filtered"); 15770fa7f681SAndrew Gallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1578adae7080SAndrew Gallatin "dropped_no_big_buffer", 1579adae7080SAndrew Gallatin CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_no_big_buffer, 15806d87a65dSAndrew Gallatin 0, mxge_handle_be32, 1581adae7080SAndrew Gallatin "I", "dropped_no_big_buffer"); 1582b2fc195eSAndrew Gallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1583b2fc195eSAndrew Gallatin "dropped_no_small_buffer", 1584b2fc195eSAndrew Gallatin CTLTYPE_INT|CTLFLAG_RD, 1585b2fc195eSAndrew Gallatin &fw->dropped_no_small_buffer, 15866d87a65dSAndrew Gallatin 0, mxge_handle_be32, 1587b2fc195eSAndrew Gallatin "I", "dropped_no_small_buffer"); 1588b2fc195eSAndrew Gallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1589adae7080SAndrew Gallatin "dropped_overrun", 1590adae7080SAndrew Gallatin CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_overrun, 15916d87a65dSAndrew Gallatin 0, mxge_handle_be32, 1592adae7080SAndrew Gallatin "I", "dropped_overrun"); 1593adae7080SAndrew Gallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1594adae7080SAndrew Gallatin "dropped_pause", 1595adae7080SAndrew Gallatin CTLTYPE_INT|CTLFLAG_RD, 1596adae7080SAndrew Gallatin &fw->dropped_pause, 1597adae7080SAndrew Gallatin 0, mxge_handle_be32, 1598adae7080SAndrew Gallatin "I", "dropped_pause"); 1599adae7080SAndrew Gallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1600adae7080SAndrew Gallatin "dropped_runt", 1601adae7080SAndrew Gallatin CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_runt, 1602adae7080SAndrew Gallatin 0, mxge_handle_be32, 1603adae7080SAndrew Gallatin "I", "dropped_runt"); 1604b2fc195eSAndrew Gallatin 1605a0394e33SAndrew Gallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1606a0394e33SAndrew Gallatin "dropped_unicast_filtered", 1607a0394e33SAndrew Gallatin CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_unicast_filtered, 1608a0394e33SAndrew Gallatin 0, mxge_handle_be32, 1609a0394e33SAndrew Gallatin "I", "dropped_unicast_filtered"); 1610a0394e33SAndrew Gallatin 16115e7d8541SAndrew Gallatin /* verbose printing? */ 1612b2fc195eSAndrew Gallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 16135e7d8541SAndrew Gallatin "verbose", 16145e7d8541SAndrew Gallatin CTLFLAG_RW, &mxge_verbose, 16155e7d8541SAndrew Gallatin 0, "verbose printing"); 1616b2fc195eSAndrew Gallatin 16171e413cf9SAndrew Gallatin /* add counters exported for debugging from all slices */ 16181e413cf9SAndrew Gallatin sysctl_ctx_init(&sc->slice_sysctl_ctx); 16191e413cf9SAndrew Gallatin sc->slice_sysctl_tree = 16201e413cf9SAndrew Gallatin SYSCTL_ADD_NODE(&sc->slice_sysctl_ctx, children, OID_AUTO, 16211e413cf9SAndrew Gallatin "slice", CTLFLAG_RD, 0, ""); 16221e413cf9SAndrew Gallatin 16231e413cf9SAndrew Gallatin for (slice = 0; slice < sc->num_slices; slice++) { 16241e413cf9SAndrew Gallatin ss = &sc->ss[slice]; 16251e413cf9SAndrew Gallatin sysctl_ctx_init(&ss->sysctl_ctx); 16261e413cf9SAndrew Gallatin ctx = &ss->sysctl_ctx; 16271e413cf9SAndrew Gallatin children = SYSCTL_CHILDREN(sc->slice_sysctl_tree); 16281e413cf9SAndrew Gallatin sprintf(slice_num, "%d", slice); 16291e413cf9SAndrew Gallatin ss->sysctl_tree = 16301e413cf9SAndrew Gallatin SYSCTL_ADD_NODE(ctx, children, OID_AUTO, slice_num, 16311e413cf9SAndrew Gallatin CTLFLAG_RD, 0, ""); 16321e413cf9SAndrew Gallatin children = SYSCTL_CHILDREN(ss->sysctl_tree); 1633053e637fSAndrew Gallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 16341e413cf9SAndrew Gallatin "rx_small_cnt", 16351e413cf9SAndrew Gallatin CTLFLAG_RD, &ss->rx_small.cnt, 16361e413cf9SAndrew Gallatin 0, "rx_small_cnt"); 16371e413cf9SAndrew Gallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 16381e413cf9SAndrew Gallatin "rx_big_cnt", 16391e413cf9SAndrew Gallatin CTLFLAG_RD, &ss->rx_big.cnt, 16401e413cf9SAndrew Gallatin 0, "rx_small_cnt"); 1641e936121dSHans Petter Selasky SYSCTL_ADD_U64(ctx, children, OID_AUTO, 164226dd49c6SAndrew Gallatin "lro_flushed", CTLFLAG_RD, &ss->lc.lro_flushed, 1643053e637fSAndrew Gallatin 0, "number of lro merge queues flushed"); 1644053e637fSAndrew Gallatin 1645e936121dSHans Petter Selasky SYSCTL_ADD_U64(ctx, children, OID_AUTO, 164626dd49c6SAndrew Gallatin "lro_bad_csum", CTLFLAG_RD, &ss->lc.lro_bad_csum, 164726dd49c6SAndrew Gallatin 0, "number of bad csums preventing LRO"); 164826dd49c6SAndrew Gallatin 1649e936121dSHans Petter Selasky SYSCTL_ADD_U64(ctx, children, OID_AUTO, 165026dd49c6SAndrew Gallatin "lro_queued", CTLFLAG_RD, &ss->lc.lro_queued, 16511e413cf9SAndrew Gallatin 0, "number of frames appended to lro merge" 16521e413cf9SAndrew Gallatin "queues"); 1653053e637fSAndrew Gallatin 1654c6cb3e3fSAndrew Gallatin #ifndef IFNET_BUF_RING 16551e413cf9SAndrew Gallatin /* only transmit from slice 0 for now */ 16561e413cf9SAndrew Gallatin if (slice > 0) 16571e413cf9SAndrew Gallatin continue; 1658c6cb3e3fSAndrew Gallatin #endif 1659c6cb3e3fSAndrew Gallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1660c6cb3e3fSAndrew Gallatin "tx_req", 1661c6cb3e3fSAndrew Gallatin CTLFLAG_RD, &ss->tx.req, 1662c6cb3e3fSAndrew Gallatin 0, "tx_req"); 16631e413cf9SAndrew Gallatin 16641e413cf9SAndrew Gallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 16651e413cf9SAndrew Gallatin "tx_done", 16661e413cf9SAndrew Gallatin CTLFLAG_RD, &ss->tx.done, 16671e413cf9SAndrew Gallatin 0, "tx_done"); 16681e413cf9SAndrew Gallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 16691e413cf9SAndrew Gallatin "tx_pkt_done", 16701e413cf9SAndrew Gallatin CTLFLAG_RD, &ss->tx.pkt_done, 16711e413cf9SAndrew Gallatin 0, "tx_done"); 16721e413cf9SAndrew Gallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 16731e413cf9SAndrew Gallatin "tx_stall", 16741e413cf9SAndrew Gallatin CTLFLAG_RD, &ss->tx.stall, 16751e413cf9SAndrew Gallatin 0, "tx_stall"); 16761e413cf9SAndrew Gallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 16771e413cf9SAndrew Gallatin "tx_wake", 16781e413cf9SAndrew Gallatin CTLFLAG_RD, &ss->tx.wake, 16791e413cf9SAndrew Gallatin 0, "tx_wake"); 16801e413cf9SAndrew Gallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 16811e413cf9SAndrew Gallatin "tx_defrag", 16821e413cf9SAndrew Gallatin CTLFLAG_RD, &ss->tx.defrag, 16831e413cf9SAndrew Gallatin 0, "tx_defrag"); 1684c6cb3e3fSAndrew Gallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1685c6cb3e3fSAndrew Gallatin "tx_queue_active", 1686c6cb3e3fSAndrew Gallatin CTLFLAG_RD, &ss->tx.queue_active, 1687c6cb3e3fSAndrew Gallatin 0, "tx_queue_active"); 1688c6cb3e3fSAndrew Gallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1689c6cb3e3fSAndrew Gallatin "tx_activate", 1690c6cb3e3fSAndrew Gallatin CTLFLAG_RD, &ss->tx.activate, 1691c6cb3e3fSAndrew Gallatin 0, "tx_activate"); 1692c6cb3e3fSAndrew Gallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1693c6cb3e3fSAndrew Gallatin "tx_deactivate", 1694c6cb3e3fSAndrew Gallatin CTLFLAG_RD, &ss->tx.deactivate, 1695c6cb3e3fSAndrew Gallatin 0, "tx_deactivate"); 16961e413cf9SAndrew Gallatin } 1697b2fc195eSAndrew Gallatin } 1698b2fc195eSAndrew Gallatin 1699b2fc195eSAndrew Gallatin /* copy an array of mcp_kreq_ether_send_t's to the mcp. Copy 1700b2fc195eSAndrew Gallatin backwards one at a time and handle ring wraps */ 1701b2fc195eSAndrew Gallatin 1702b2fc195eSAndrew Gallatin static inline void 17031e413cf9SAndrew Gallatin mxge_submit_req_backwards(mxge_tx_ring_t *tx, 1704b2fc195eSAndrew Gallatin mcp_kreq_ether_send_t *src, int cnt) 1705b2fc195eSAndrew Gallatin { 1706b2fc195eSAndrew Gallatin int idx, starting_slot; 1707b2fc195eSAndrew Gallatin starting_slot = tx->req; 1708b2fc195eSAndrew Gallatin while (cnt > 1) { 1709b2fc195eSAndrew Gallatin cnt--; 1710b2fc195eSAndrew Gallatin idx = (starting_slot + cnt) & tx->mask; 17116d87a65dSAndrew Gallatin mxge_pio_copy(&tx->lanai[idx], 1712b2fc195eSAndrew Gallatin &src[cnt], sizeof(*src)); 171373c7c83fSAndrew Gallatin wmb(); 1714b2fc195eSAndrew Gallatin } 1715b2fc195eSAndrew Gallatin } 1716b2fc195eSAndrew Gallatin 1717b2fc195eSAndrew Gallatin /* 1718b2fc195eSAndrew Gallatin * copy an array of mcp_kreq_ether_send_t's to the mcp. Copy 1719b2fc195eSAndrew Gallatin * at most 32 bytes at a time, so as to avoid involving the software 1720b2fc195eSAndrew Gallatin * pio handler in the nic. We re-write the first segment's flags 1721b2fc195eSAndrew Gallatin * to mark them valid only after writing the entire chain 1722b2fc195eSAndrew Gallatin */ 1723b2fc195eSAndrew Gallatin 1724b2fc195eSAndrew Gallatin static inline void 17251e413cf9SAndrew Gallatin mxge_submit_req(mxge_tx_ring_t *tx, mcp_kreq_ether_send_t *src, 1726b2fc195eSAndrew Gallatin int cnt) 1727b2fc195eSAndrew Gallatin { 1728b2fc195eSAndrew Gallatin int idx, i; 1729b2fc195eSAndrew Gallatin uint32_t *src_ints; 1730b2fc195eSAndrew Gallatin volatile uint32_t *dst_ints; 1731b2fc195eSAndrew Gallatin mcp_kreq_ether_send_t *srcp; 1732b2fc195eSAndrew Gallatin volatile mcp_kreq_ether_send_t *dstp, *dst; 17335e7d8541SAndrew Gallatin uint8_t last_flags; 1734b2fc195eSAndrew Gallatin 1735b2fc195eSAndrew Gallatin idx = tx->req & tx->mask; 1736b2fc195eSAndrew Gallatin 17375e7d8541SAndrew Gallatin last_flags = src->flags; 17385e7d8541SAndrew Gallatin src->flags = 0; 173973c7c83fSAndrew Gallatin wmb(); 1740b2fc195eSAndrew Gallatin dst = dstp = &tx->lanai[idx]; 1741b2fc195eSAndrew Gallatin srcp = src; 1742b2fc195eSAndrew Gallatin 1743b2fc195eSAndrew Gallatin if ((idx + cnt) < tx->mask) { 1744b2fc195eSAndrew Gallatin for (i = 0; i < (cnt - 1); i += 2) { 17456d87a65dSAndrew Gallatin mxge_pio_copy(dstp, srcp, 2 * sizeof(*src)); 174673c7c83fSAndrew Gallatin wmb(); /* force write every 32 bytes */ 1747b2fc195eSAndrew Gallatin srcp += 2; 1748b2fc195eSAndrew Gallatin dstp += 2; 1749b2fc195eSAndrew Gallatin } 1750b2fc195eSAndrew Gallatin } else { 1751b2fc195eSAndrew Gallatin /* submit all but the first request, and ensure 1752b2fc195eSAndrew Gallatin that it is submitted below */ 17536d87a65dSAndrew Gallatin mxge_submit_req_backwards(tx, src, cnt); 1754b2fc195eSAndrew Gallatin i = 0; 1755b2fc195eSAndrew Gallatin } 1756b2fc195eSAndrew Gallatin if (i < cnt) { 1757b2fc195eSAndrew Gallatin /* submit the first request */ 17586d87a65dSAndrew Gallatin mxge_pio_copy(dstp, srcp, sizeof(*src)); 175973c7c83fSAndrew Gallatin wmb(); /* barrier before setting valid flag */ 1760b2fc195eSAndrew Gallatin } 1761b2fc195eSAndrew Gallatin 1762b2fc195eSAndrew Gallatin /* re-write the last 32-bits with the valid flags */ 17635e7d8541SAndrew Gallatin src->flags = last_flags; 1764b2fc195eSAndrew Gallatin src_ints = (uint32_t *)src; 1765b2fc195eSAndrew Gallatin src_ints+=3; 1766b2fc195eSAndrew Gallatin dst_ints = (volatile uint32_t *)dst; 1767b2fc195eSAndrew Gallatin dst_ints+=3; 1768b2fc195eSAndrew Gallatin *dst_ints = *src_ints; 1769b2fc195eSAndrew Gallatin tx->req += cnt; 177073c7c83fSAndrew Gallatin wmb(); 1771b2fc195eSAndrew Gallatin } 1772b2fc195eSAndrew Gallatin 17730a7a780eSAndrew Gallatin static int 17740a7a780eSAndrew Gallatin mxge_parse_tx(struct mxge_slice_state *ss, struct mbuf *m, 17750a7a780eSAndrew Gallatin struct mxge_pkt_info *pi) 17760a7a780eSAndrew Gallatin { 17770a7a780eSAndrew Gallatin struct ether_vlan_header *eh; 17780a7a780eSAndrew Gallatin uint16_t etype; 17790a7a780eSAndrew Gallatin int tso = m->m_pkthdr.csum_flags & (CSUM_TSO); 17800a7a780eSAndrew Gallatin #if IFCAP_TSO6 && defined(INET6) 17810a7a780eSAndrew Gallatin int nxt; 17820a7a780eSAndrew Gallatin #endif 17830a7a780eSAndrew Gallatin 17840a7a780eSAndrew Gallatin eh = mtod(m, struct ether_vlan_header *); 17850a7a780eSAndrew Gallatin if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 17860a7a780eSAndrew Gallatin etype = ntohs(eh->evl_proto); 17870a7a780eSAndrew Gallatin pi->ip_off = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 17880a7a780eSAndrew Gallatin } else { 17890a7a780eSAndrew Gallatin etype = ntohs(eh->evl_encap_proto); 17900a7a780eSAndrew Gallatin pi->ip_off = ETHER_HDR_LEN; 17910a7a780eSAndrew Gallatin } 17920a7a780eSAndrew Gallatin 17930a7a780eSAndrew Gallatin switch (etype) { 17940a7a780eSAndrew Gallatin case ETHERTYPE_IP: 17950a7a780eSAndrew Gallatin /* 17960a7a780eSAndrew Gallatin * ensure ip header is in first mbuf, copy it to a 17970a7a780eSAndrew Gallatin * scratch buffer if not 17980a7a780eSAndrew Gallatin */ 17990a7a780eSAndrew Gallatin pi->ip = (struct ip *)(m->m_data + pi->ip_off); 18000a7a780eSAndrew Gallatin pi->ip6 = NULL; 18010a7a780eSAndrew Gallatin if (__predict_false(m->m_len < pi->ip_off + sizeof(*pi->ip))) { 18020a7a780eSAndrew Gallatin m_copydata(m, 0, pi->ip_off + sizeof(*pi->ip), 18030a7a780eSAndrew Gallatin ss->scratch); 18040a7a780eSAndrew Gallatin pi->ip = (struct ip *)(ss->scratch + pi->ip_off); 18050a7a780eSAndrew Gallatin } 18060a7a780eSAndrew Gallatin pi->ip_hlen = pi->ip->ip_hl << 2; 18070a7a780eSAndrew Gallatin if (!tso) 18080a7a780eSAndrew Gallatin return 0; 18090a7a780eSAndrew Gallatin 18100a7a780eSAndrew Gallatin if (__predict_false(m->m_len < pi->ip_off + pi->ip_hlen + 18110a7a780eSAndrew Gallatin sizeof(struct tcphdr))) { 18120a7a780eSAndrew Gallatin m_copydata(m, 0, pi->ip_off + pi->ip_hlen + 18130a7a780eSAndrew Gallatin sizeof(struct tcphdr), ss->scratch); 18140a7a780eSAndrew Gallatin pi->ip = (struct ip *)(ss->scratch + pi->ip_off); 18150a7a780eSAndrew Gallatin } 18160a7a780eSAndrew Gallatin pi->tcp = (struct tcphdr *)((char *)pi->ip + pi->ip_hlen); 18170a7a780eSAndrew Gallatin break; 18180a7a780eSAndrew Gallatin #if IFCAP_TSO6 && defined(INET6) 18190a7a780eSAndrew Gallatin case ETHERTYPE_IPV6: 18200a7a780eSAndrew Gallatin pi->ip6 = (struct ip6_hdr *)(m->m_data + pi->ip_off); 18210a7a780eSAndrew Gallatin if (__predict_false(m->m_len < pi->ip_off + sizeof(*pi->ip6))) { 18220a7a780eSAndrew Gallatin m_copydata(m, 0, pi->ip_off + sizeof(*pi->ip6), 18230a7a780eSAndrew Gallatin ss->scratch); 18240a7a780eSAndrew Gallatin pi->ip6 = (struct ip6_hdr *)(ss->scratch + pi->ip_off); 18250a7a780eSAndrew Gallatin } 18260a7a780eSAndrew Gallatin nxt = 0; 18270a7a780eSAndrew Gallatin pi->ip_hlen = ip6_lasthdr(m, pi->ip_off, IPPROTO_IPV6, &nxt); 18280a7a780eSAndrew Gallatin pi->ip_hlen -= pi->ip_off; 18290a7a780eSAndrew Gallatin if (nxt != IPPROTO_TCP && nxt != IPPROTO_UDP) 18300a7a780eSAndrew Gallatin return EINVAL; 18310a7a780eSAndrew Gallatin 18320a7a780eSAndrew Gallatin if (!tso) 18330a7a780eSAndrew Gallatin return 0; 18340a7a780eSAndrew Gallatin 18350a7a780eSAndrew Gallatin if (pi->ip_off + pi->ip_hlen > ss->sc->max_tso6_hlen) 18360a7a780eSAndrew Gallatin return EINVAL; 18370a7a780eSAndrew Gallatin 18380a7a780eSAndrew Gallatin if (__predict_false(m->m_len < pi->ip_off + pi->ip_hlen + 18390a7a780eSAndrew Gallatin sizeof(struct tcphdr))) { 18400a7a780eSAndrew Gallatin m_copydata(m, 0, pi->ip_off + pi->ip_hlen + 18410a7a780eSAndrew Gallatin sizeof(struct tcphdr), ss->scratch); 18420a7a780eSAndrew Gallatin pi->ip6 = (struct ip6_hdr *)(ss->scratch + pi->ip_off); 18430a7a780eSAndrew Gallatin } 18440a7a780eSAndrew Gallatin pi->tcp = (struct tcphdr *)((char *)pi->ip6 + pi->ip_hlen); 18450a7a780eSAndrew Gallatin break; 18460a7a780eSAndrew Gallatin #endif 18470a7a780eSAndrew Gallatin default: 18480a7a780eSAndrew Gallatin return EINVAL; 18490a7a780eSAndrew Gallatin } 18500a7a780eSAndrew Gallatin return 0; 18510a7a780eSAndrew Gallatin } 18520a7a780eSAndrew Gallatin 185337d89b0cSAndrew Gallatin #if IFCAP_TSO4 185437d89b0cSAndrew Gallatin 1855b2fc195eSAndrew Gallatin static void 18561e413cf9SAndrew Gallatin mxge_encap_tso(struct mxge_slice_state *ss, struct mbuf *m, 18570a7a780eSAndrew Gallatin int busdma_seg_cnt, struct mxge_pkt_info *pi) 1858aed8e389SAndrew Gallatin { 18591e413cf9SAndrew Gallatin mxge_tx_ring_t *tx; 1860aed8e389SAndrew Gallatin mcp_kreq_ether_send_t *req; 1861aed8e389SAndrew Gallatin bus_dma_segment_t *seg; 1862aed8e389SAndrew Gallatin uint32_t low, high_swapped; 1863aed8e389SAndrew Gallatin int len, seglen, cum_len, cum_len_next; 1864aed8e389SAndrew Gallatin int next_is_first, chop, cnt, rdma_count, small; 18650a7a780eSAndrew Gallatin uint16_t pseudo_hdr_offset, cksum_offset, mss, sum; 1866aed8e389SAndrew Gallatin uint8_t flags, flags_next; 1867aed8e389SAndrew Gallatin static int once; 1868aed8e389SAndrew Gallatin 1869aed8e389SAndrew Gallatin mss = m->m_pkthdr.tso_segsz; 1870aed8e389SAndrew Gallatin 1871aed8e389SAndrew Gallatin /* negative cum_len signifies to the 1872aed8e389SAndrew Gallatin * send loop that we are still in the 1873aed8e389SAndrew Gallatin * header portion of the TSO packet. 1874aed8e389SAndrew Gallatin */ 1875aed8e389SAndrew Gallatin 18760a7a780eSAndrew Gallatin cksum_offset = pi->ip_off + pi->ip_hlen; 18770a7a780eSAndrew Gallatin cum_len = -(cksum_offset + (pi->tcp->th_off << 2)); 1878aed8e389SAndrew Gallatin 1879aed8e389SAndrew Gallatin /* TSO implies checksum offload on this hardware */ 18800a7a780eSAndrew Gallatin if (__predict_false((m->m_pkthdr.csum_flags & (CSUM_TCP|CSUM_TCP_IPV6)) == 0)) { 18814ed8ca8fSAndrew Gallatin /* 18824ed8ca8fSAndrew Gallatin * If packet has full TCP csum, replace it with pseudo hdr 18834ed8ca8fSAndrew Gallatin * sum that the NIC expects, otherwise the NIC will emit 18844ed8ca8fSAndrew Gallatin * packets with bad TCP checksums. 18854ed8ca8fSAndrew Gallatin */ 18864ed8ca8fSAndrew Gallatin m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); 18870a7a780eSAndrew Gallatin if (pi->ip6) { 18880a7a780eSAndrew Gallatin #if (CSUM_TCP_IPV6 != 0) && defined(INET6) 18890a7a780eSAndrew Gallatin m->m_pkthdr.csum_flags |= CSUM_TCP_IPV6; 18900a7a780eSAndrew Gallatin sum = in6_cksum_pseudo(pi->ip6, 18910a7a780eSAndrew Gallatin m->m_pkthdr.len - cksum_offset, 18920a7a780eSAndrew Gallatin IPPROTO_TCP, 0); 18930a7a780eSAndrew Gallatin #endif 18940a7a780eSAndrew Gallatin } else { 1895abc5b96bSAndrew Gallatin #ifdef INET 18960a7a780eSAndrew Gallatin m->m_pkthdr.csum_flags |= CSUM_TCP; 18970a7a780eSAndrew Gallatin sum = in_pseudo(pi->ip->ip_src.s_addr, 18980a7a780eSAndrew Gallatin pi->ip->ip_dst.s_addr, 18990a7a780eSAndrew Gallatin htons(IPPROTO_TCP + (m->m_pkthdr.len - 19000a7a780eSAndrew Gallatin cksum_offset))); 1901abc5b96bSAndrew Gallatin #endif 19020a7a780eSAndrew Gallatin } 19030a7a780eSAndrew Gallatin m_copyback(m, offsetof(struct tcphdr, th_sum) + 19040a7a780eSAndrew Gallatin cksum_offset, sizeof(sum), (caddr_t)&sum); 19054ed8ca8fSAndrew Gallatin } 1906aed8e389SAndrew Gallatin flags = MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST; 1907aed8e389SAndrew Gallatin 1908aed8e389SAndrew Gallatin 1909aed8e389SAndrew Gallatin /* for TSO, pseudo_hdr_offset holds mss. 1910aed8e389SAndrew Gallatin * The firmware figures out where to put 1911aed8e389SAndrew Gallatin * the checksum by parsing the header. */ 1912aed8e389SAndrew Gallatin pseudo_hdr_offset = htobe16(mss); 1913aed8e389SAndrew Gallatin 19140a7a780eSAndrew Gallatin if (pi->ip6) { 19150a7a780eSAndrew Gallatin /* 19160a7a780eSAndrew Gallatin * for IPv6 TSO, the "checksum offset" is re-purposed 19170a7a780eSAndrew Gallatin * to store the TCP header len 19180a7a780eSAndrew Gallatin */ 19190a7a780eSAndrew Gallatin cksum_offset = (pi->tcp->th_off << 2); 19200a7a780eSAndrew Gallatin } 19210a7a780eSAndrew Gallatin 19221e413cf9SAndrew Gallatin tx = &ss->tx; 1923aed8e389SAndrew Gallatin req = tx->req_list; 1924aed8e389SAndrew Gallatin seg = tx->seg_list; 1925aed8e389SAndrew Gallatin cnt = 0; 1926aed8e389SAndrew Gallatin rdma_count = 0; 1927aed8e389SAndrew Gallatin /* "rdma_count" is the number of RDMAs belonging to the 1928aed8e389SAndrew Gallatin * current packet BEFORE the current send request. For 1929aed8e389SAndrew Gallatin * non-TSO packets, this is equal to "count". 1930aed8e389SAndrew Gallatin * For TSO packets, rdma_count needs to be reset 1931aed8e389SAndrew Gallatin * to 0 after a segment cut. 1932aed8e389SAndrew Gallatin * 1933aed8e389SAndrew Gallatin * The rdma_count field of the send request is 1934aed8e389SAndrew Gallatin * the number of RDMAs of the packet starting at 1935aed8e389SAndrew Gallatin * that request. For TSO send requests with one ore more cuts 1936aed8e389SAndrew Gallatin * in the middle, this is the number of RDMAs starting 1937aed8e389SAndrew Gallatin * after the last cut in the request. All previous 1938aed8e389SAndrew Gallatin * segments before the last cut implicitly have 1 RDMA. 1939aed8e389SAndrew Gallatin * 1940aed8e389SAndrew Gallatin * Since the number of RDMAs is not known beforehand, 1941aed8e389SAndrew Gallatin * it must be filled-in retroactively - after each 1942aed8e389SAndrew Gallatin * segmentation cut or at the end of the entire packet. 1943aed8e389SAndrew Gallatin */ 1944aed8e389SAndrew Gallatin 1945aed8e389SAndrew Gallatin while (busdma_seg_cnt) { 1946aed8e389SAndrew Gallatin /* Break the busdma segment up into pieces*/ 1947aed8e389SAndrew Gallatin low = MXGE_LOWPART_TO_U32(seg->ds_addr); 1948aed8e389SAndrew Gallatin high_swapped = htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr)); 1949e39a0a37SAndrew Gallatin len = seg->ds_len; 1950aed8e389SAndrew Gallatin 1951aed8e389SAndrew Gallatin while (len) { 1952aed8e389SAndrew Gallatin flags_next = flags & ~MXGEFW_FLAGS_FIRST; 1953e39a0a37SAndrew Gallatin seglen = len; 1954aed8e389SAndrew Gallatin cum_len_next = cum_len + seglen; 1955aed8e389SAndrew Gallatin (req-rdma_count)->rdma_count = rdma_count + 1; 1956aed8e389SAndrew Gallatin if (__predict_true(cum_len >= 0)) { 1957aed8e389SAndrew Gallatin /* payload */ 1958aed8e389SAndrew Gallatin chop = (cum_len_next > mss); 1959aed8e389SAndrew Gallatin cum_len_next = cum_len_next % mss; 1960aed8e389SAndrew Gallatin next_is_first = (cum_len_next == 0); 1961aed8e389SAndrew Gallatin flags |= chop * MXGEFW_FLAGS_TSO_CHOP; 1962aed8e389SAndrew Gallatin flags_next |= next_is_first * 1963aed8e389SAndrew Gallatin MXGEFW_FLAGS_FIRST; 1964aed8e389SAndrew Gallatin rdma_count |= -(chop | next_is_first); 1965aed8e389SAndrew Gallatin rdma_count += chop & !next_is_first; 1966aed8e389SAndrew Gallatin } else if (cum_len_next >= 0) { 1967aed8e389SAndrew Gallatin /* header ends */ 1968aed8e389SAndrew Gallatin rdma_count = -1; 1969aed8e389SAndrew Gallatin cum_len_next = 0; 1970aed8e389SAndrew Gallatin seglen = -cum_len; 1971aed8e389SAndrew Gallatin small = (mss <= MXGEFW_SEND_SMALL_SIZE); 1972aed8e389SAndrew Gallatin flags_next = MXGEFW_FLAGS_TSO_PLD | 1973aed8e389SAndrew Gallatin MXGEFW_FLAGS_FIRST | 1974aed8e389SAndrew Gallatin (small * MXGEFW_FLAGS_SMALL); 1975aed8e389SAndrew Gallatin } 1976aed8e389SAndrew Gallatin 1977aed8e389SAndrew Gallatin req->addr_high = high_swapped; 1978aed8e389SAndrew Gallatin req->addr_low = htobe32(low); 1979aed8e389SAndrew Gallatin req->pseudo_hdr_offset = pseudo_hdr_offset; 1980aed8e389SAndrew Gallatin req->pad = 0; 1981aed8e389SAndrew Gallatin req->rdma_count = 1; 1982aed8e389SAndrew Gallatin req->length = htobe16(seglen); 1983aed8e389SAndrew Gallatin req->cksum_offset = cksum_offset; 1984aed8e389SAndrew Gallatin req->flags = flags | ((cum_len & 1) * 1985aed8e389SAndrew Gallatin MXGEFW_FLAGS_ALIGN_ODD); 1986aed8e389SAndrew Gallatin low += seglen; 1987aed8e389SAndrew Gallatin len -= seglen; 1988aed8e389SAndrew Gallatin cum_len = cum_len_next; 1989aed8e389SAndrew Gallatin flags = flags_next; 1990aed8e389SAndrew Gallatin req++; 1991aed8e389SAndrew Gallatin cnt++; 1992aed8e389SAndrew Gallatin rdma_count++; 19930a7a780eSAndrew Gallatin if (cksum_offset != 0 && !pi->ip6) { 1994aed8e389SAndrew Gallatin if (__predict_false(cksum_offset > seglen)) 1995aed8e389SAndrew Gallatin cksum_offset -= seglen; 1996aed8e389SAndrew Gallatin else 1997aed8e389SAndrew Gallatin cksum_offset = 0; 19980a7a780eSAndrew Gallatin } 1999adae7080SAndrew Gallatin if (__predict_false(cnt > tx->max_desc)) 2000aed8e389SAndrew Gallatin goto drop; 2001aed8e389SAndrew Gallatin } 2002aed8e389SAndrew Gallatin busdma_seg_cnt--; 2003aed8e389SAndrew Gallatin seg++; 2004aed8e389SAndrew Gallatin } 2005aed8e389SAndrew Gallatin (req-rdma_count)->rdma_count = rdma_count; 2006aed8e389SAndrew Gallatin 2007aed8e389SAndrew Gallatin do { 2008aed8e389SAndrew Gallatin req--; 2009aed8e389SAndrew Gallatin req->flags |= MXGEFW_FLAGS_TSO_LAST; 2010aed8e389SAndrew Gallatin } while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP | MXGEFW_FLAGS_FIRST))); 2011aed8e389SAndrew Gallatin 2012aed8e389SAndrew Gallatin tx->info[((cnt - 1) + tx->req) & tx->mask].flag = 1; 2013aed8e389SAndrew Gallatin mxge_submit_req(tx, tx->req_list, cnt); 2014c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING 2015c6cb3e3fSAndrew Gallatin if ((ss->sc->num_slices > 1) && tx->queue_active == 0) { 2016c6cb3e3fSAndrew Gallatin /* tell the NIC to start polling this slice */ 2017c6cb3e3fSAndrew Gallatin *tx->send_go = 1; 2018c6cb3e3fSAndrew Gallatin tx->queue_active = 1; 2019c6cb3e3fSAndrew Gallatin tx->activate++; 2020c6cb3e3fSAndrew Gallatin wmb(); 2021c6cb3e3fSAndrew Gallatin } 2022c6cb3e3fSAndrew Gallatin #endif 2023aed8e389SAndrew Gallatin return; 2024aed8e389SAndrew Gallatin 2025aed8e389SAndrew Gallatin drop: 2026e39a0a37SAndrew Gallatin bus_dmamap_unload(tx->dmat, tx->info[tx->req & tx->mask].map); 2027aed8e389SAndrew Gallatin m_freem(m); 2028c6cb3e3fSAndrew Gallatin ss->oerrors++; 2029aed8e389SAndrew Gallatin if (!once) { 2030adae7080SAndrew Gallatin printf("tx->max_desc exceeded via TSO!\n"); 2031adae7080SAndrew Gallatin printf("mss = %d, %ld, %d!\n", mss, 2032adae7080SAndrew Gallatin (long)seg - (long)tx->seg_list, tx->max_desc); 2033aed8e389SAndrew Gallatin once = 1; 2034aed8e389SAndrew Gallatin } 2035aed8e389SAndrew Gallatin return; 2036aed8e389SAndrew Gallatin 2037aed8e389SAndrew Gallatin } 2038aed8e389SAndrew Gallatin 203937d89b0cSAndrew Gallatin #endif /* IFCAP_TSO4 */ 204037d89b0cSAndrew Gallatin 204137d89b0cSAndrew Gallatin #ifdef MXGE_NEW_VLAN_API 2042c792928fSAndrew Gallatin /* 2043c792928fSAndrew Gallatin * We reproduce the software vlan tag insertion from 2044c792928fSAndrew Gallatin * net/if_vlan.c:vlan_start() here so that we can advertise "hardware" 2045c792928fSAndrew Gallatin * vlan tag insertion. We need to advertise this in order to have the 2046c792928fSAndrew Gallatin * vlan interface respect our csum offload flags. 2047c792928fSAndrew Gallatin */ 2048c792928fSAndrew Gallatin static struct mbuf * 2049c792928fSAndrew Gallatin mxge_vlan_tag_insert(struct mbuf *m) 2050c792928fSAndrew Gallatin { 2051c792928fSAndrew Gallatin struct ether_vlan_header *evl; 2052c792928fSAndrew Gallatin 2053c6499eccSGleb Smirnoff M_PREPEND(m, ETHER_VLAN_ENCAP_LEN, M_NOWAIT); 2054c792928fSAndrew Gallatin if (__predict_false(m == NULL)) 2055c792928fSAndrew Gallatin return NULL; 2056c792928fSAndrew Gallatin if (m->m_len < sizeof(*evl)) { 2057c792928fSAndrew Gallatin m = m_pullup(m, sizeof(*evl)); 2058c792928fSAndrew Gallatin if (__predict_false(m == NULL)) 2059c792928fSAndrew Gallatin return NULL; 2060c792928fSAndrew Gallatin } 2061c792928fSAndrew Gallatin /* 2062c792928fSAndrew Gallatin * Transform the Ethernet header into an Ethernet header 2063c792928fSAndrew Gallatin * with 802.1Q encapsulation. 2064c792928fSAndrew Gallatin */ 2065c792928fSAndrew Gallatin evl = mtod(m, struct ether_vlan_header *); 2066c792928fSAndrew Gallatin bcopy((char *)evl + ETHER_VLAN_ENCAP_LEN, 2067c792928fSAndrew Gallatin (char *)evl, ETHER_HDR_LEN - ETHER_TYPE_LEN); 2068c792928fSAndrew Gallatin evl->evl_encap_proto = htons(ETHERTYPE_VLAN); 2069c792928fSAndrew Gallatin evl->evl_tag = htons(m->m_pkthdr.ether_vtag); 2070c792928fSAndrew Gallatin m->m_flags &= ~M_VLANTAG; 2071c792928fSAndrew Gallatin return m; 2072c792928fSAndrew Gallatin } 207337d89b0cSAndrew Gallatin #endif /* MXGE_NEW_VLAN_API */ 2074c792928fSAndrew Gallatin 2075aed8e389SAndrew Gallatin static void 20761e413cf9SAndrew Gallatin mxge_encap(struct mxge_slice_state *ss, struct mbuf *m) 2077b2fc195eSAndrew Gallatin { 20780a7a780eSAndrew Gallatin struct mxge_pkt_info pi = {0,0,0,0}; 20791e413cf9SAndrew Gallatin mxge_softc_t *sc; 2080b2fc195eSAndrew Gallatin mcp_kreq_ether_send_t *req; 2081b2fc195eSAndrew Gallatin bus_dma_segment_t *seg; 2082b2fc195eSAndrew Gallatin struct mbuf *m_tmp; 2083b2fc195eSAndrew Gallatin struct ifnet *ifp; 20841e413cf9SAndrew Gallatin mxge_tx_ring_t *tx; 20850a7a780eSAndrew Gallatin int cnt, cum_len, err, i, idx, odd_flag; 2086aed8e389SAndrew Gallatin uint16_t pseudo_hdr_offset; 2087aed8e389SAndrew Gallatin uint8_t flags, cksum_offset; 2088b2fc195eSAndrew Gallatin 2089b2fc195eSAndrew Gallatin 20901e413cf9SAndrew Gallatin sc = ss->sc; 2091b2fc195eSAndrew Gallatin ifp = sc->ifp; 20921e413cf9SAndrew Gallatin tx = &ss->tx; 2093b2fc195eSAndrew Gallatin 209437d89b0cSAndrew Gallatin #ifdef MXGE_NEW_VLAN_API 2095c792928fSAndrew Gallatin if (m->m_flags & M_VLANTAG) { 2096c792928fSAndrew Gallatin m = mxge_vlan_tag_insert(m); 2097c792928fSAndrew Gallatin if (__predict_false(m == NULL)) 20980a7a780eSAndrew Gallatin goto drop_without_m; 2099c792928fSAndrew Gallatin } 210037d89b0cSAndrew Gallatin #endif 21010a7a780eSAndrew Gallatin if (m->m_pkthdr.csum_flags & 21020a7a780eSAndrew Gallatin (CSUM_TSO | CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6)) { 21030a7a780eSAndrew Gallatin if (mxge_parse_tx(ss, m, &pi)) 21040a7a780eSAndrew Gallatin goto drop; 21050a7a780eSAndrew Gallatin } 21060a7a780eSAndrew Gallatin 2107b2fc195eSAndrew Gallatin /* (try to) map the frame for DMA */ 2108b2fc195eSAndrew Gallatin idx = tx->req & tx->mask; 2109b2fc195eSAndrew Gallatin err = bus_dmamap_load_mbuf_sg(tx->dmat, tx->info[idx].map, 2110aed8e389SAndrew Gallatin m, tx->seg_list, &cnt, 2111b2fc195eSAndrew Gallatin BUS_DMA_NOWAIT); 2112adae7080SAndrew Gallatin if (__predict_false(err == EFBIG)) { 2113b2fc195eSAndrew Gallatin /* Too many segments in the chain. Try 2114b2fc195eSAndrew Gallatin to defrag */ 2115b2fc195eSAndrew Gallatin m_tmp = m_defrag(m, M_NOWAIT); 2116b2fc195eSAndrew Gallatin if (m_tmp == NULL) { 2117b2fc195eSAndrew Gallatin goto drop; 2118b2fc195eSAndrew Gallatin } 21191e413cf9SAndrew Gallatin ss->tx.defrag++; 2120b2fc195eSAndrew Gallatin m = m_tmp; 2121b2fc195eSAndrew Gallatin err = bus_dmamap_load_mbuf_sg(tx->dmat, 2122b2fc195eSAndrew Gallatin tx->info[idx].map, 2123aed8e389SAndrew Gallatin m, tx->seg_list, &cnt, 2124b2fc195eSAndrew Gallatin BUS_DMA_NOWAIT); 2125b2fc195eSAndrew Gallatin } 2126adae7080SAndrew Gallatin if (__predict_false(err != 0)) { 2127aed8e389SAndrew Gallatin device_printf(sc->dev, "bus_dmamap_load_mbuf_sg returned %d" 2128aed8e389SAndrew Gallatin " packet len = %d\n", err, m->m_pkthdr.len); 2129b2fc195eSAndrew Gallatin goto drop; 2130b2fc195eSAndrew Gallatin } 2131b2fc195eSAndrew Gallatin bus_dmamap_sync(tx->dmat, tx->info[idx].map, 2132b2fc195eSAndrew Gallatin BUS_DMASYNC_PREWRITE); 21335e7d8541SAndrew Gallatin tx->info[idx].m = m; 2134b2fc195eSAndrew Gallatin 213537d89b0cSAndrew Gallatin #if IFCAP_TSO4 2136aed8e389SAndrew Gallatin /* TSO is different enough, we handle it in another routine */ 2137aed8e389SAndrew Gallatin if (m->m_pkthdr.csum_flags & (CSUM_TSO)) { 21380a7a780eSAndrew Gallatin mxge_encap_tso(ss, m, cnt, &pi); 2139aed8e389SAndrew Gallatin return; 2140aed8e389SAndrew Gallatin } 214137d89b0cSAndrew Gallatin #endif 2142aed8e389SAndrew Gallatin 2143b2fc195eSAndrew Gallatin req = tx->req_list; 2144b2fc195eSAndrew Gallatin cksum_offset = 0; 21455e7d8541SAndrew Gallatin pseudo_hdr_offset = 0; 21465e7d8541SAndrew Gallatin flags = MXGEFW_FLAGS_NO_TSO; 2147b2fc195eSAndrew Gallatin 2148b2fc195eSAndrew Gallatin /* checksum offloading? */ 21490a7a780eSAndrew Gallatin if (m->m_pkthdr.csum_flags & 21500a7a780eSAndrew Gallatin (CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6)) { 2151aed8e389SAndrew Gallatin /* ensure ip header is in first mbuf, copy 2152aed8e389SAndrew Gallatin it to a scratch buffer if not */ 21530a7a780eSAndrew Gallatin cksum_offset = pi.ip_off + pi.ip_hlen; 2154b2fc195eSAndrew Gallatin pseudo_hdr_offset = cksum_offset + m->m_pkthdr.csum_data; 21555e7d8541SAndrew Gallatin pseudo_hdr_offset = htobe16(pseudo_hdr_offset); 2156b2fc195eSAndrew Gallatin req->cksum_offset = cksum_offset; 21575e7d8541SAndrew Gallatin flags |= MXGEFW_FLAGS_CKSUM; 2158aed8e389SAndrew Gallatin odd_flag = MXGEFW_FLAGS_ALIGN_ODD; 2159aed8e389SAndrew Gallatin } else { 2160aed8e389SAndrew Gallatin odd_flag = 0; 2161b2fc195eSAndrew Gallatin } 21625e7d8541SAndrew Gallatin if (m->m_pkthdr.len < MXGEFW_SEND_SMALL_SIZE) 21635e7d8541SAndrew Gallatin flags |= MXGEFW_FLAGS_SMALL; 2164b2fc195eSAndrew Gallatin 2165b2fc195eSAndrew Gallatin /* convert segments into a request list */ 2166b2fc195eSAndrew Gallatin cum_len = 0; 2167aed8e389SAndrew Gallatin seg = tx->seg_list; 21685e7d8541SAndrew Gallatin req->flags = MXGEFW_FLAGS_FIRST; 2169b2fc195eSAndrew Gallatin for (i = 0; i < cnt; i++) { 2170b2fc195eSAndrew Gallatin req->addr_low = 21716d87a65dSAndrew Gallatin htobe32(MXGE_LOWPART_TO_U32(seg->ds_addr)); 2172b2fc195eSAndrew Gallatin req->addr_high = 21736d87a65dSAndrew Gallatin htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr)); 2174b2fc195eSAndrew Gallatin req->length = htobe16(seg->ds_len); 2175b2fc195eSAndrew Gallatin req->cksum_offset = cksum_offset; 2176b2fc195eSAndrew Gallatin if (cksum_offset > seg->ds_len) 2177b2fc195eSAndrew Gallatin cksum_offset -= seg->ds_len; 2178b2fc195eSAndrew Gallatin else 2179b2fc195eSAndrew Gallatin cksum_offset = 0; 21805e7d8541SAndrew Gallatin req->pseudo_hdr_offset = pseudo_hdr_offset; 21815e7d8541SAndrew Gallatin req->pad = 0; /* complete solid 16-byte block */ 21825e7d8541SAndrew Gallatin req->rdma_count = 1; 2183aed8e389SAndrew Gallatin req->flags |= flags | ((cum_len & 1) * odd_flag); 2184b2fc195eSAndrew Gallatin cum_len += seg->ds_len; 2185b2fc195eSAndrew Gallatin seg++; 2186b2fc195eSAndrew Gallatin req++; 2187b2fc195eSAndrew Gallatin req->flags = 0; 2188b2fc195eSAndrew Gallatin } 2189b2fc195eSAndrew Gallatin req--; 2190b2fc195eSAndrew Gallatin /* pad runts to 60 bytes */ 2191b2fc195eSAndrew Gallatin if (cum_len < 60) { 2192b2fc195eSAndrew Gallatin req++; 2193b2fc195eSAndrew Gallatin req->addr_low = 21946d87a65dSAndrew Gallatin htobe32(MXGE_LOWPART_TO_U32(sc->zeropad_dma.bus_addr)); 2195b2fc195eSAndrew Gallatin req->addr_high = 21966d87a65dSAndrew Gallatin htobe32(MXGE_HIGHPART_TO_U32(sc->zeropad_dma.bus_addr)); 2197b2fc195eSAndrew Gallatin req->length = htobe16(60 - cum_len); 21985e7d8541SAndrew Gallatin req->cksum_offset = 0; 21995e7d8541SAndrew Gallatin req->pseudo_hdr_offset = pseudo_hdr_offset; 22005e7d8541SAndrew Gallatin req->pad = 0; /* complete solid 16-byte block */ 22015e7d8541SAndrew Gallatin req->rdma_count = 1; 2202aed8e389SAndrew Gallatin req->flags |= flags | ((cum_len & 1) * odd_flag); 2203b2fc195eSAndrew Gallatin cnt++; 2204b2fc195eSAndrew Gallatin } 22055e7d8541SAndrew Gallatin 22065e7d8541SAndrew Gallatin tx->req_list[0].rdma_count = cnt; 22075e7d8541SAndrew Gallatin #if 0 22085e7d8541SAndrew Gallatin /* print what the firmware will see */ 22095e7d8541SAndrew Gallatin for (i = 0; i < cnt; i++) { 22105e7d8541SAndrew Gallatin printf("%d: addr: 0x%x 0x%x len:%d pso%d," 22115e7d8541SAndrew Gallatin "cso:%d, flags:0x%x, rdma:%d\n", 22125e7d8541SAndrew Gallatin i, (int)ntohl(tx->req_list[i].addr_high), 22135e7d8541SAndrew Gallatin (int)ntohl(tx->req_list[i].addr_low), 22145e7d8541SAndrew Gallatin (int)ntohs(tx->req_list[i].length), 22155e7d8541SAndrew Gallatin (int)ntohs(tx->req_list[i].pseudo_hdr_offset), 22165e7d8541SAndrew Gallatin tx->req_list[i].cksum_offset, tx->req_list[i].flags, 22175e7d8541SAndrew Gallatin tx->req_list[i].rdma_count); 22185e7d8541SAndrew Gallatin } 22195e7d8541SAndrew Gallatin printf("--------------\n"); 22205e7d8541SAndrew Gallatin #endif 22215e7d8541SAndrew Gallatin tx->info[((cnt - 1) + tx->req) & tx->mask].flag = 1; 22226d87a65dSAndrew Gallatin mxge_submit_req(tx, tx->req_list, cnt); 2223c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING 2224c6cb3e3fSAndrew Gallatin if ((ss->sc->num_slices > 1) && tx->queue_active == 0) { 2225c6cb3e3fSAndrew Gallatin /* tell the NIC to start polling this slice */ 2226c6cb3e3fSAndrew Gallatin *tx->send_go = 1; 2227c6cb3e3fSAndrew Gallatin tx->queue_active = 1; 2228c6cb3e3fSAndrew Gallatin tx->activate++; 2229c6cb3e3fSAndrew Gallatin wmb(); 2230c6cb3e3fSAndrew Gallatin } 2231c6cb3e3fSAndrew Gallatin #endif 2232b2fc195eSAndrew Gallatin return; 2233b2fc195eSAndrew Gallatin 2234b2fc195eSAndrew Gallatin drop: 2235b2fc195eSAndrew Gallatin m_freem(m); 22360a7a780eSAndrew Gallatin drop_without_m: 2237c6cb3e3fSAndrew Gallatin ss->oerrors++; 2238b2fc195eSAndrew Gallatin return; 2239b2fc195eSAndrew Gallatin } 2240b2fc195eSAndrew Gallatin 2241c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING 2242c6cb3e3fSAndrew Gallatin static void 2243c6cb3e3fSAndrew Gallatin mxge_qflush(struct ifnet *ifp) 2244c6cb3e3fSAndrew Gallatin { 2245c6cb3e3fSAndrew Gallatin mxge_softc_t *sc = ifp->if_softc; 2246c6cb3e3fSAndrew Gallatin mxge_tx_ring_t *tx; 2247c6cb3e3fSAndrew Gallatin struct mbuf *m; 2248c6cb3e3fSAndrew Gallatin int slice; 2249b2fc195eSAndrew Gallatin 2250c6cb3e3fSAndrew Gallatin for (slice = 0; slice < sc->num_slices; slice++) { 2251c6cb3e3fSAndrew Gallatin tx = &sc->ss[slice].tx; 2252c6cb3e3fSAndrew Gallatin mtx_lock(&tx->mtx); 2253c6cb3e3fSAndrew Gallatin while ((m = buf_ring_dequeue_sc(tx->br)) != NULL) 2254c6cb3e3fSAndrew Gallatin m_freem(m); 2255c6cb3e3fSAndrew Gallatin mtx_unlock(&tx->mtx); 2256c6cb3e3fSAndrew Gallatin } 2257c6cb3e3fSAndrew Gallatin if_qflush(ifp); 2258c6cb3e3fSAndrew Gallatin } 22596d914a32SAndrew Gallatin 2260c6cb3e3fSAndrew Gallatin static inline void 2261c6cb3e3fSAndrew Gallatin mxge_start_locked(struct mxge_slice_state *ss) 2262c6cb3e3fSAndrew Gallatin { 2263c6cb3e3fSAndrew Gallatin mxge_softc_t *sc; 2264c6cb3e3fSAndrew Gallatin struct mbuf *m; 2265c6cb3e3fSAndrew Gallatin struct ifnet *ifp; 2266c6cb3e3fSAndrew Gallatin mxge_tx_ring_t *tx; 2267c6cb3e3fSAndrew Gallatin 2268c6cb3e3fSAndrew Gallatin sc = ss->sc; 2269c6cb3e3fSAndrew Gallatin ifp = sc->ifp; 2270c6cb3e3fSAndrew Gallatin tx = &ss->tx; 2271c6cb3e3fSAndrew Gallatin 2272c6cb3e3fSAndrew Gallatin while ((tx->mask - (tx->req - tx->done)) > tx->max_desc) { 2273c6cb3e3fSAndrew Gallatin m = drbr_dequeue(ifp, tx->br); 2274c6cb3e3fSAndrew Gallatin if (m == NULL) { 2275c6cb3e3fSAndrew Gallatin return; 2276c6cb3e3fSAndrew Gallatin } 2277c6cb3e3fSAndrew Gallatin /* let BPF see it */ 2278c6cb3e3fSAndrew Gallatin BPF_MTAP(ifp, m); 2279c6cb3e3fSAndrew Gallatin 2280c6cb3e3fSAndrew Gallatin /* give it to the nic */ 2281c6cb3e3fSAndrew Gallatin mxge_encap(ss, m); 2282c6cb3e3fSAndrew Gallatin } 2283c6cb3e3fSAndrew Gallatin /* ran out of transmit slots */ 2284c6cb3e3fSAndrew Gallatin if (((ss->if_drv_flags & IFF_DRV_OACTIVE) == 0) 2285c6cb3e3fSAndrew Gallatin && (!drbr_empty(ifp, tx->br))) { 2286c6cb3e3fSAndrew Gallatin ss->if_drv_flags |= IFF_DRV_OACTIVE; 2287c6cb3e3fSAndrew Gallatin tx->stall++; 2288c6cb3e3fSAndrew Gallatin } 2289c6cb3e3fSAndrew Gallatin } 2290c6cb3e3fSAndrew Gallatin 2291c6cb3e3fSAndrew Gallatin static int 2292c6cb3e3fSAndrew Gallatin mxge_transmit_locked(struct mxge_slice_state *ss, struct mbuf *m) 2293c6cb3e3fSAndrew Gallatin { 2294c6cb3e3fSAndrew Gallatin mxge_softc_t *sc; 2295c6cb3e3fSAndrew Gallatin struct ifnet *ifp; 2296c6cb3e3fSAndrew Gallatin mxge_tx_ring_t *tx; 2297c6cb3e3fSAndrew Gallatin int err; 2298c6cb3e3fSAndrew Gallatin 2299c6cb3e3fSAndrew Gallatin sc = ss->sc; 2300c6cb3e3fSAndrew Gallatin ifp = sc->ifp; 2301c6cb3e3fSAndrew Gallatin tx = &ss->tx; 2302c6cb3e3fSAndrew Gallatin 2303c6cb3e3fSAndrew Gallatin if ((ss->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 2304c6cb3e3fSAndrew Gallatin IFF_DRV_RUNNING) { 2305c6cb3e3fSAndrew Gallatin err = drbr_enqueue(ifp, tx->br, m); 2306c6cb3e3fSAndrew Gallatin return (err); 2307c6cb3e3fSAndrew Gallatin } 2308c6cb3e3fSAndrew Gallatin 2309193cbc4dSMax Laier if (!drbr_needs_enqueue(ifp, tx->br) && 2310c6cb3e3fSAndrew Gallatin ((tx->mask - (tx->req - tx->done)) > tx->max_desc)) { 2311c6cb3e3fSAndrew Gallatin /* let BPF see it */ 2312c6cb3e3fSAndrew Gallatin BPF_MTAP(ifp, m); 2313c6cb3e3fSAndrew Gallatin /* give it to the nic */ 2314c6cb3e3fSAndrew Gallatin mxge_encap(ss, m); 2315c6cb3e3fSAndrew Gallatin } else if ((err = drbr_enqueue(ifp, tx->br, m)) != 0) { 2316c6cb3e3fSAndrew Gallatin return (err); 2317c6cb3e3fSAndrew Gallatin } 2318c6cb3e3fSAndrew Gallatin if (!drbr_empty(ifp, tx->br)) 2319c6cb3e3fSAndrew Gallatin mxge_start_locked(ss); 2320c6cb3e3fSAndrew Gallatin return (0); 2321c6cb3e3fSAndrew Gallatin } 2322c6cb3e3fSAndrew Gallatin 2323c6cb3e3fSAndrew Gallatin static int 2324c6cb3e3fSAndrew Gallatin mxge_transmit(struct ifnet *ifp, struct mbuf *m) 2325c6cb3e3fSAndrew Gallatin { 2326c6cb3e3fSAndrew Gallatin mxge_softc_t *sc = ifp->if_softc; 2327c6cb3e3fSAndrew Gallatin struct mxge_slice_state *ss; 2328c6cb3e3fSAndrew Gallatin mxge_tx_ring_t *tx; 2329c6cb3e3fSAndrew Gallatin int err = 0; 2330c6cb3e3fSAndrew Gallatin int slice; 2331c6cb3e3fSAndrew Gallatin 2332c6cb3e3fSAndrew Gallatin slice = m->m_pkthdr.flowid; 2333c6cb3e3fSAndrew Gallatin slice &= (sc->num_slices - 1); /* num_slices always power of 2 */ 2334c6cb3e3fSAndrew Gallatin 2335c6cb3e3fSAndrew Gallatin ss = &sc->ss[slice]; 2336c6cb3e3fSAndrew Gallatin tx = &ss->tx; 2337c6cb3e3fSAndrew Gallatin 2338c6cb3e3fSAndrew Gallatin if (mtx_trylock(&tx->mtx)) { 2339c6cb3e3fSAndrew Gallatin err = mxge_transmit_locked(ss, m); 2340c6cb3e3fSAndrew Gallatin mtx_unlock(&tx->mtx); 2341c6cb3e3fSAndrew Gallatin } else { 2342c6cb3e3fSAndrew Gallatin err = drbr_enqueue(ifp, tx->br, m); 2343c6cb3e3fSAndrew Gallatin } 2344c6cb3e3fSAndrew Gallatin 2345c6cb3e3fSAndrew Gallatin return (err); 2346c6cb3e3fSAndrew Gallatin } 2347c6cb3e3fSAndrew Gallatin 2348c6cb3e3fSAndrew Gallatin #else 23496d914a32SAndrew Gallatin 23506d914a32SAndrew Gallatin static inline void 23511e413cf9SAndrew Gallatin mxge_start_locked(struct mxge_slice_state *ss) 2352b2fc195eSAndrew Gallatin { 23531e413cf9SAndrew Gallatin mxge_softc_t *sc; 2354b2fc195eSAndrew Gallatin struct mbuf *m; 2355b2fc195eSAndrew Gallatin struct ifnet *ifp; 23561e413cf9SAndrew Gallatin mxge_tx_ring_t *tx; 2357b2fc195eSAndrew Gallatin 23581e413cf9SAndrew Gallatin sc = ss->sc; 2359b2fc195eSAndrew Gallatin ifp = sc->ifp; 23601e413cf9SAndrew Gallatin tx = &ss->tx; 2361adae7080SAndrew Gallatin while ((tx->mask - (tx->req - tx->done)) > tx->max_desc) { 23626d914a32SAndrew Gallatin IFQ_DRV_DEQUEUE(&ifp->if_snd, m); 23636d914a32SAndrew Gallatin if (m == NULL) { 23646d914a32SAndrew Gallatin return; 23656d914a32SAndrew Gallatin } 2366b2fc195eSAndrew Gallatin /* let BPF see it */ 2367b2fc195eSAndrew Gallatin BPF_MTAP(ifp, m); 2368b2fc195eSAndrew Gallatin 2369b2fc195eSAndrew Gallatin /* give it to the nic */ 23701e413cf9SAndrew Gallatin mxge_encap(ss, m); 23716d914a32SAndrew Gallatin } 23726d914a32SAndrew Gallatin /* ran out of transmit slots */ 2373a82c2581SAndrew Gallatin if ((sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) { 2374b2fc195eSAndrew Gallatin sc->ifp->if_drv_flags |= IFF_DRV_OACTIVE; 2375adae7080SAndrew Gallatin tx->stall++; 2376a82c2581SAndrew Gallatin } 2377b2fc195eSAndrew Gallatin } 2378c6cb3e3fSAndrew Gallatin #endif 2379b2fc195eSAndrew Gallatin static void 23806d87a65dSAndrew Gallatin mxge_start(struct ifnet *ifp) 2381b2fc195eSAndrew Gallatin { 23826d87a65dSAndrew Gallatin mxge_softc_t *sc = ifp->if_softc; 23831e413cf9SAndrew Gallatin struct mxge_slice_state *ss; 2384b2fc195eSAndrew Gallatin 23851e413cf9SAndrew Gallatin /* only use the first slice for now */ 23861e413cf9SAndrew Gallatin ss = &sc->ss[0]; 23871e413cf9SAndrew Gallatin mtx_lock(&ss->tx.mtx); 23881e413cf9SAndrew Gallatin mxge_start_locked(ss); 23891e413cf9SAndrew Gallatin mtx_unlock(&ss->tx.mtx); 2390b2fc195eSAndrew Gallatin } 2391b2fc195eSAndrew Gallatin 23925e7d8541SAndrew Gallatin /* 23935e7d8541SAndrew Gallatin * copy an array of mcp_kreq_ether_recv_t's to the mcp. Copy 23945e7d8541SAndrew Gallatin * at most 32 bytes at a time, so as to avoid involving the software 23955e7d8541SAndrew Gallatin * pio handler in the nic. We re-write the first segment's low 23965e7d8541SAndrew Gallatin * DMA address to mark it valid only after we write the entire chunk 23975e7d8541SAndrew Gallatin * in a burst 23985e7d8541SAndrew Gallatin */ 23995e7d8541SAndrew Gallatin static inline void 24005e7d8541SAndrew Gallatin mxge_submit_8rx(volatile mcp_kreq_ether_recv_t *dst, 24015e7d8541SAndrew Gallatin mcp_kreq_ether_recv_t *src) 24025e7d8541SAndrew Gallatin { 24035e7d8541SAndrew Gallatin uint32_t low; 24045e7d8541SAndrew Gallatin 24055e7d8541SAndrew Gallatin low = src->addr_low; 24065e7d8541SAndrew Gallatin src->addr_low = 0xffffffff; 2407a1480dfbSAndrew Gallatin mxge_pio_copy(dst, src, 4 * sizeof (*src)); 240873c7c83fSAndrew Gallatin wmb(); 2409a1480dfbSAndrew Gallatin mxge_pio_copy(dst + 4, src + 4, 4 * sizeof (*src)); 241073c7c83fSAndrew Gallatin wmb(); 241140385a5fSAndrew Gallatin src->addr_low = low; 24125e7d8541SAndrew Gallatin dst->addr_low = low; 241373c7c83fSAndrew Gallatin wmb(); 24145e7d8541SAndrew Gallatin } 24155e7d8541SAndrew Gallatin 2416b2fc195eSAndrew Gallatin static int 24171e413cf9SAndrew Gallatin mxge_get_buf_small(struct mxge_slice_state *ss, bus_dmamap_t map, int idx) 2418b2fc195eSAndrew Gallatin { 2419b2fc195eSAndrew Gallatin bus_dma_segment_t seg; 2420b2fc195eSAndrew Gallatin struct mbuf *m; 24211e413cf9SAndrew Gallatin mxge_rx_ring_t *rx = &ss->rx_small; 2422b2fc195eSAndrew Gallatin int cnt, err; 2423b2fc195eSAndrew Gallatin 2424c6499eccSGleb Smirnoff m = m_gethdr(M_NOWAIT, MT_DATA); 2425b2fc195eSAndrew Gallatin if (m == NULL) { 2426b2fc195eSAndrew Gallatin rx->alloc_fail++; 2427b2fc195eSAndrew Gallatin err = ENOBUFS; 2428b2fc195eSAndrew Gallatin goto done; 2429b2fc195eSAndrew Gallatin } 2430b2fc195eSAndrew Gallatin m->m_len = MHLEN; 2431b2fc195eSAndrew Gallatin err = bus_dmamap_load_mbuf_sg(rx->dmat, map, m, 2432b2fc195eSAndrew Gallatin &seg, &cnt, BUS_DMA_NOWAIT); 2433b2fc195eSAndrew Gallatin if (err != 0) { 2434b2fc195eSAndrew Gallatin m_free(m); 2435b2fc195eSAndrew Gallatin goto done; 2436b2fc195eSAndrew Gallatin } 2437b2fc195eSAndrew Gallatin rx->info[idx].m = m; 2438b2fc195eSAndrew Gallatin rx->shadow[idx].addr_low = 24396d87a65dSAndrew Gallatin htobe32(MXGE_LOWPART_TO_U32(seg.ds_addr)); 2440b2fc195eSAndrew Gallatin rx->shadow[idx].addr_high = 24416d87a65dSAndrew Gallatin htobe32(MXGE_HIGHPART_TO_U32(seg.ds_addr)); 2442b2fc195eSAndrew Gallatin 2443b2fc195eSAndrew Gallatin done: 2444adae7080SAndrew Gallatin if ((idx & 7) == 7) 2445adae7080SAndrew Gallatin mxge_submit_8rx(&rx->lanai[idx - 7], &rx->shadow[idx - 7]); 2446b2fc195eSAndrew Gallatin return err; 2447b2fc195eSAndrew Gallatin } 2448b2fc195eSAndrew Gallatin 2449b2fc195eSAndrew Gallatin static int 24501e413cf9SAndrew Gallatin mxge_get_buf_big(struct mxge_slice_state *ss, bus_dmamap_t map, int idx) 2451b2fc195eSAndrew Gallatin { 2452053e637fSAndrew Gallatin bus_dma_segment_t seg[3]; 2453b2fc195eSAndrew Gallatin struct mbuf *m; 24541e413cf9SAndrew Gallatin mxge_rx_ring_t *rx = &ss->rx_big; 2455053e637fSAndrew Gallatin int cnt, err, i; 2456b2fc195eSAndrew Gallatin 2457c6499eccSGleb Smirnoff m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rx->cl_size); 2458b2fc195eSAndrew Gallatin if (m == NULL) { 2459b2fc195eSAndrew Gallatin rx->alloc_fail++; 2460b2fc195eSAndrew Gallatin err = ENOBUFS; 2461b2fc195eSAndrew Gallatin goto done; 2462b2fc195eSAndrew Gallatin } 24634d9a5852SAndrew Gallatin m->m_len = rx->mlen; 2464b2fc195eSAndrew Gallatin err = bus_dmamap_load_mbuf_sg(rx->dmat, map, m, 2465053e637fSAndrew Gallatin seg, &cnt, BUS_DMA_NOWAIT); 2466b2fc195eSAndrew Gallatin if (err != 0) { 2467b2fc195eSAndrew Gallatin m_free(m); 2468b2fc195eSAndrew Gallatin goto done; 2469b2fc195eSAndrew Gallatin } 2470b2fc195eSAndrew Gallatin rx->info[idx].m = m; 2471b0f7b922SAndrew Gallatin rx->shadow[idx].addr_low = 2472b0f7b922SAndrew Gallatin htobe32(MXGE_LOWPART_TO_U32(seg->ds_addr)); 2473b0f7b922SAndrew Gallatin rx->shadow[idx].addr_high = 2474b0f7b922SAndrew Gallatin htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr)); 2475053e637fSAndrew Gallatin 2476b0f7b922SAndrew Gallatin #if MXGE_VIRT_JUMBOS 2477b0f7b922SAndrew Gallatin for (i = 1; i < cnt; i++) { 2478053e637fSAndrew Gallatin rx->shadow[idx + i].addr_low = 2479053e637fSAndrew Gallatin htobe32(MXGE_LOWPART_TO_U32(seg[i].ds_addr)); 2480053e637fSAndrew Gallatin rx->shadow[idx + i].addr_high = 2481053e637fSAndrew Gallatin htobe32(MXGE_HIGHPART_TO_U32(seg[i].ds_addr)); 2482053e637fSAndrew Gallatin } 2483b0f7b922SAndrew Gallatin #endif 2484b2fc195eSAndrew Gallatin 2485b2fc195eSAndrew Gallatin done: 2486053e637fSAndrew Gallatin for (i = 0; i < rx->nbufs; i++) { 2487b2fc195eSAndrew Gallatin if ((idx & 7) == 7) { 24885e7d8541SAndrew Gallatin mxge_submit_8rx(&rx->lanai[idx - 7], 24895e7d8541SAndrew Gallatin &rx->shadow[idx - 7]); 2490b2fc195eSAndrew Gallatin } 2491053e637fSAndrew Gallatin idx++; 2492053e637fSAndrew Gallatin } 2493b2fc195eSAndrew Gallatin return err; 2494b2fc195eSAndrew Gallatin } 2495b2fc195eSAndrew Gallatin 249626dd49c6SAndrew Gallatin #ifdef INET6 249726dd49c6SAndrew Gallatin 249826dd49c6SAndrew Gallatin static uint16_t 249926dd49c6SAndrew Gallatin mxge_csum_generic(uint16_t *raw, int len) 250026dd49c6SAndrew Gallatin { 250126dd49c6SAndrew Gallatin uint32_t csum; 250226dd49c6SAndrew Gallatin 250326dd49c6SAndrew Gallatin 250426dd49c6SAndrew Gallatin csum = 0; 250526dd49c6SAndrew Gallatin while (len > 0) { 250626dd49c6SAndrew Gallatin csum += *raw; 250726dd49c6SAndrew Gallatin raw++; 250826dd49c6SAndrew Gallatin len -= 2; 250926dd49c6SAndrew Gallatin } 251026dd49c6SAndrew Gallatin csum = (csum >> 16) + (csum & 0xffff); 251126dd49c6SAndrew Gallatin csum = (csum >> 16) + (csum & 0xffff); 251226dd49c6SAndrew Gallatin return (uint16_t)csum; 251326dd49c6SAndrew Gallatin } 251426dd49c6SAndrew Gallatin 251526dd49c6SAndrew Gallatin static inline uint16_t 251626dd49c6SAndrew Gallatin mxge_rx_csum6(void *p, struct mbuf *m, uint32_t csum) 251726dd49c6SAndrew Gallatin { 251826dd49c6SAndrew Gallatin uint32_t partial; 251926dd49c6SAndrew Gallatin int nxt, cksum_offset; 252026dd49c6SAndrew Gallatin struct ip6_hdr *ip6 = p; 252126dd49c6SAndrew Gallatin uint16_t c; 252226dd49c6SAndrew Gallatin 252326dd49c6SAndrew Gallatin nxt = ip6->ip6_nxt; 252426dd49c6SAndrew Gallatin cksum_offset = sizeof (*ip6) + ETHER_HDR_LEN; 252526dd49c6SAndrew Gallatin if (nxt != IPPROTO_TCP && nxt != IPPROTO_UDP) { 252626dd49c6SAndrew Gallatin cksum_offset = ip6_lasthdr(m, ETHER_HDR_LEN, 252726dd49c6SAndrew Gallatin IPPROTO_IPV6, &nxt); 252826dd49c6SAndrew Gallatin if (nxt != IPPROTO_TCP && nxt != IPPROTO_UDP) 252926dd49c6SAndrew Gallatin return (1); 253026dd49c6SAndrew Gallatin } 253126dd49c6SAndrew Gallatin 253226dd49c6SAndrew Gallatin /* 253326dd49c6SAndrew Gallatin * IPv6 headers do not contain a checksum, and hence 253426dd49c6SAndrew Gallatin * do not checksum to zero, so they don't "fall out" 253526dd49c6SAndrew Gallatin * of the partial checksum calculation like IPv4 253626dd49c6SAndrew Gallatin * headers do. We need to fix the partial checksum by 253726dd49c6SAndrew Gallatin * subtracting the checksum of the IPv6 header. 253826dd49c6SAndrew Gallatin */ 253926dd49c6SAndrew Gallatin 254026dd49c6SAndrew Gallatin partial = mxge_csum_generic((uint16_t *)ip6, cksum_offset - 254126dd49c6SAndrew Gallatin ETHER_HDR_LEN); 254226dd49c6SAndrew Gallatin csum += ~partial; 254326dd49c6SAndrew Gallatin csum += (csum < ~partial); 254426dd49c6SAndrew Gallatin csum = (csum >> 16) + (csum & 0xFFFF); 254526dd49c6SAndrew Gallatin csum = (csum >> 16) + (csum & 0xFFFF); 254626dd49c6SAndrew Gallatin c = in6_cksum_pseudo(ip6, m->m_pkthdr.len - cksum_offset, nxt, 254726dd49c6SAndrew Gallatin csum); 254826dd49c6SAndrew Gallatin c ^= 0xffff; 254926dd49c6SAndrew Gallatin return (c); 255026dd49c6SAndrew Gallatin } 255126dd49c6SAndrew Gallatin #endif /* INET6 */ 25529b03b0f3SAndrew Gallatin /* 25539b03b0f3SAndrew Gallatin * Myri10GE hardware checksums are not valid if the sender 25549b03b0f3SAndrew Gallatin * padded the frame with non-zero padding. This is because 25559b03b0f3SAndrew Gallatin * the firmware just does a simple 16-bit 1s complement 25569b03b0f3SAndrew Gallatin * checksum across the entire frame, excluding the first 14 2557053e637fSAndrew Gallatin * bytes. It is best to simply to check the checksum and 2558053e637fSAndrew Gallatin * tell the stack about it only if the checksum is good 25599b03b0f3SAndrew Gallatin */ 25609b03b0f3SAndrew Gallatin 2561053e637fSAndrew Gallatin static inline uint16_t 2562053e637fSAndrew Gallatin mxge_rx_csum(struct mbuf *m, int csum) 2563053e637fSAndrew Gallatin { 2564053e637fSAndrew Gallatin struct ether_header *eh; 256526dd49c6SAndrew Gallatin #ifdef INET 2566053e637fSAndrew Gallatin struct ip *ip; 256726dd49c6SAndrew Gallatin #endif 2568abc5b96bSAndrew Gallatin #if defined(INET) || defined(INET6) 2569abc5b96bSAndrew Gallatin int cap = m->m_pkthdr.rcvif->if_capenable; 2570abc5b96bSAndrew Gallatin #endif 257126dd49c6SAndrew Gallatin uint16_t c, etype; 257226dd49c6SAndrew Gallatin 2573053e637fSAndrew Gallatin 2574053e637fSAndrew Gallatin eh = mtod(m, struct ether_header *); 257526dd49c6SAndrew Gallatin etype = ntohs(eh->ether_type); 257626dd49c6SAndrew Gallatin switch (etype) { 2577eb6219e3SAndrew Gallatin #ifdef INET 257826dd49c6SAndrew Gallatin case ETHERTYPE_IP: 257926dd49c6SAndrew Gallatin if ((cap & IFCAP_RXCSUM) == 0) 258026dd49c6SAndrew Gallatin return (1); 258126dd49c6SAndrew Gallatin ip = (struct ip *)(eh + 1); 258226dd49c6SAndrew Gallatin if (ip->ip_p != IPPROTO_TCP && ip->ip_p != IPPROTO_UDP) 258326dd49c6SAndrew Gallatin return (1); 2584053e637fSAndrew Gallatin c = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, 258526dd49c6SAndrew Gallatin htonl(ntohs(csum) + ntohs(ip->ip_len) - 258626dd49c6SAndrew Gallatin (ip->ip_hl << 2) + ip->ip_p)); 2587053e637fSAndrew Gallatin c ^= 0xffff; 258826dd49c6SAndrew Gallatin break; 258926dd49c6SAndrew Gallatin #endif 259026dd49c6SAndrew Gallatin #ifdef INET6 259126dd49c6SAndrew Gallatin case ETHERTYPE_IPV6: 259226dd49c6SAndrew Gallatin if ((cap & IFCAP_RXCSUM_IPV6) == 0) 259326dd49c6SAndrew Gallatin return (1); 259426dd49c6SAndrew Gallatin c = mxge_rx_csum6((eh + 1), m, csum); 259526dd49c6SAndrew Gallatin break; 259626dd49c6SAndrew Gallatin #endif 259726dd49c6SAndrew Gallatin default: 259826dd49c6SAndrew Gallatin c = 1; 259926dd49c6SAndrew Gallatin } 2600053e637fSAndrew Gallatin return (c); 26015e7d8541SAndrew Gallatin } 2602053e637fSAndrew Gallatin 2603c792928fSAndrew Gallatin static void 2604c792928fSAndrew Gallatin mxge_vlan_tag_remove(struct mbuf *m, uint32_t *csum) 2605c792928fSAndrew Gallatin { 2606c792928fSAndrew Gallatin struct ether_vlan_header *evl; 2607c792928fSAndrew Gallatin struct ether_header *eh; 2608c792928fSAndrew Gallatin uint32_t partial; 2609c792928fSAndrew Gallatin 2610c792928fSAndrew Gallatin evl = mtod(m, struct ether_vlan_header *); 2611c792928fSAndrew Gallatin eh = mtod(m, struct ether_header *); 2612c792928fSAndrew Gallatin 2613c792928fSAndrew Gallatin /* 2614c792928fSAndrew Gallatin * fix checksum by subtracting ETHER_VLAN_ENCAP_LEN bytes 2615c792928fSAndrew Gallatin * after what the firmware thought was the end of the ethernet 2616c792928fSAndrew Gallatin * header. 2617c792928fSAndrew Gallatin */ 2618c792928fSAndrew Gallatin 2619c792928fSAndrew Gallatin /* put checksum into host byte order */ 2620c792928fSAndrew Gallatin *csum = ntohs(*csum); 2621c792928fSAndrew Gallatin partial = ntohl(*(uint32_t *)(mtod(m, char *) + ETHER_HDR_LEN)); 2622c792928fSAndrew Gallatin (*csum) += ~partial; 2623c792928fSAndrew Gallatin (*csum) += ((*csum) < ~partial); 2624c792928fSAndrew Gallatin (*csum) = ((*csum) >> 16) + ((*csum) & 0xFFFF); 2625c792928fSAndrew Gallatin (*csum) = ((*csum) >> 16) + ((*csum) & 0xFFFF); 2626c792928fSAndrew Gallatin 2627c792928fSAndrew Gallatin /* restore checksum to network byte order; 2628c792928fSAndrew Gallatin later consumers expect this */ 2629c792928fSAndrew Gallatin *csum = htons(*csum); 2630c792928fSAndrew Gallatin 2631c792928fSAndrew Gallatin /* save the tag */ 263237d89b0cSAndrew Gallatin #ifdef MXGE_NEW_VLAN_API 2633c792928fSAndrew Gallatin m->m_pkthdr.ether_vtag = ntohs(evl->evl_tag); 263437d89b0cSAndrew Gallatin #else 263537d89b0cSAndrew Gallatin { 263637d89b0cSAndrew Gallatin struct m_tag *mtag; 263737d89b0cSAndrew Gallatin mtag = m_tag_alloc(MTAG_VLAN, MTAG_VLAN_TAG, sizeof(u_int), 263837d89b0cSAndrew Gallatin M_NOWAIT); 263937d89b0cSAndrew Gallatin if (mtag == NULL) 264037d89b0cSAndrew Gallatin return; 264137d89b0cSAndrew Gallatin VLAN_TAG_VALUE(mtag) = ntohs(evl->evl_tag); 264237d89b0cSAndrew Gallatin m_tag_prepend(m, mtag); 264337d89b0cSAndrew Gallatin } 264437d89b0cSAndrew Gallatin 264537d89b0cSAndrew Gallatin #endif 264637d89b0cSAndrew Gallatin m->m_flags |= M_VLANTAG; 2647c792928fSAndrew Gallatin 2648c792928fSAndrew Gallatin /* 2649c792928fSAndrew Gallatin * Remove the 802.1q header by copying the Ethernet 2650c792928fSAndrew Gallatin * addresses over it and adjusting the beginning of 2651c792928fSAndrew Gallatin * the data in the mbuf. The encapsulated Ethernet 2652c792928fSAndrew Gallatin * type field is already in place. 2653c792928fSAndrew Gallatin */ 2654c792928fSAndrew Gallatin bcopy((char *)evl, (char *)evl + ETHER_VLAN_ENCAP_LEN, 2655c792928fSAndrew Gallatin ETHER_HDR_LEN - ETHER_TYPE_LEN); 2656c792928fSAndrew Gallatin m_adj(m, ETHER_VLAN_ENCAP_LEN); 2657c792928fSAndrew Gallatin } 2658c792928fSAndrew Gallatin 26595e7d8541SAndrew Gallatin 26605e7d8541SAndrew Gallatin static inline void 266126dd49c6SAndrew Gallatin mxge_rx_done_big(struct mxge_slice_state *ss, uint32_t len, 266226dd49c6SAndrew Gallatin uint32_t csum, int lro) 2663b2fc195eSAndrew Gallatin { 26641e413cf9SAndrew Gallatin mxge_softc_t *sc; 2665b2fc195eSAndrew Gallatin struct ifnet *ifp; 2666053e637fSAndrew Gallatin struct mbuf *m; 2667c792928fSAndrew Gallatin struct ether_header *eh; 26681e413cf9SAndrew Gallatin mxge_rx_ring_t *rx; 2669053e637fSAndrew Gallatin bus_dmamap_t old_map; 2670b2fc195eSAndrew Gallatin int idx; 2671b2fc195eSAndrew Gallatin 26721e413cf9SAndrew Gallatin sc = ss->sc; 2673b2fc195eSAndrew Gallatin ifp = sc->ifp; 26741e413cf9SAndrew Gallatin rx = &ss->rx_big; 2675b2fc195eSAndrew Gallatin idx = rx->cnt & rx->mask; 2676053e637fSAndrew Gallatin rx->cnt += rx->nbufs; 2677b2fc195eSAndrew Gallatin /* save a pointer to the received mbuf */ 2678b2fc195eSAndrew Gallatin m = rx->info[idx].m; 2679b2fc195eSAndrew Gallatin /* try to replace the received mbuf */ 26801e413cf9SAndrew Gallatin if (mxge_get_buf_big(ss, rx->extra_map, idx)) { 2681053e637fSAndrew Gallatin /* drop the frame -- the old mbuf is re-cycled */ 2682f3f040d9SGleb Smirnoff if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); 2683053e637fSAndrew Gallatin return; 2684b2fc195eSAndrew Gallatin } 2685053e637fSAndrew Gallatin 2686b2fc195eSAndrew Gallatin /* unmap the received buffer */ 2687b2fc195eSAndrew Gallatin old_map = rx->info[idx].map; 2688b2fc195eSAndrew Gallatin bus_dmamap_sync(rx->dmat, old_map, BUS_DMASYNC_POSTREAD); 2689b2fc195eSAndrew Gallatin bus_dmamap_unload(rx->dmat, old_map); 2690b2fc195eSAndrew Gallatin 2691b2fc195eSAndrew Gallatin /* swap the bus_dmamap_t's */ 2692b2fc195eSAndrew Gallatin rx->info[idx].map = rx->extra_map; 2693b2fc195eSAndrew Gallatin rx->extra_map = old_map; 2694b2fc195eSAndrew Gallatin 2695053e637fSAndrew Gallatin /* mcp implicitly skips 1st 2 bytes so that packet is properly 2696053e637fSAndrew Gallatin * aligned */ 26975e7d8541SAndrew Gallatin m->m_data += MXGEFW_PAD; 2698b2fc195eSAndrew Gallatin 2699053e637fSAndrew Gallatin m->m_pkthdr.rcvif = ifp; 2700053e637fSAndrew Gallatin m->m_len = m->m_pkthdr.len = len; 27011e413cf9SAndrew Gallatin ss->ipackets++; 2702c792928fSAndrew Gallatin eh = mtod(m, struct ether_header *); 2703c792928fSAndrew Gallatin if (eh->ether_type == htons(ETHERTYPE_VLAN)) { 2704c792928fSAndrew Gallatin mxge_vlan_tag_remove(m, &csum); 2705c792928fSAndrew Gallatin } 2706eacb70baSSepherosa Ziehau /* flowid only valid if RSS hashing is enabled */ 2707eacb70baSSepherosa Ziehau if (sc->num_slices > 1) { 2708eacb70baSSepherosa Ziehau m->m_pkthdr.flowid = (ss - sc->ss); 2709eacb70baSSepherosa Ziehau M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 2710eacb70baSSepherosa Ziehau } 2711b2fc195eSAndrew Gallatin /* if the checksum is valid, mark it in the mbuf header */ 271226dd49c6SAndrew Gallatin if ((ifp->if_capenable & (IFCAP_RXCSUM_IPV6 | IFCAP_RXCSUM)) && 271326dd49c6SAndrew Gallatin (0 == mxge_rx_csum(m, csum))) { 271426dd49c6SAndrew Gallatin /* Tell the stack that the checksum is good */ 2715053e637fSAndrew Gallatin m->m_pkthdr.csum_data = 0xffff; 271626dd49c6SAndrew Gallatin m->m_pkthdr.csum_flags = CSUM_PSEUDO_HDR | 271726dd49c6SAndrew Gallatin CSUM_DATA_VALID; 271826dd49c6SAndrew Gallatin 271926dd49c6SAndrew Gallatin #if defined(INET) || defined (INET6) 272026dd49c6SAndrew Gallatin if (lro && (0 == tcp_lro_rx(&ss->lc, m, 0))) 272126dd49c6SAndrew Gallatin return; 272226dd49c6SAndrew Gallatin #endif 2723b2fc195eSAndrew Gallatin } 2724053e637fSAndrew Gallatin /* pass the frame up the stack */ 2725053e637fSAndrew Gallatin (*ifp->if_input)(ifp, m); 2726b2fc195eSAndrew Gallatin } 2727b2fc195eSAndrew Gallatin 2728b2fc195eSAndrew Gallatin static inline void 272926dd49c6SAndrew Gallatin mxge_rx_done_small(struct mxge_slice_state *ss, uint32_t len, 273026dd49c6SAndrew Gallatin uint32_t csum, int lro) 2731b2fc195eSAndrew Gallatin { 27321e413cf9SAndrew Gallatin mxge_softc_t *sc; 2733b2fc195eSAndrew Gallatin struct ifnet *ifp; 2734c792928fSAndrew Gallatin struct ether_header *eh; 2735b2fc195eSAndrew Gallatin struct mbuf *m; 27361e413cf9SAndrew Gallatin mxge_rx_ring_t *rx; 2737b2fc195eSAndrew Gallatin bus_dmamap_t old_map; 2738b2fc195eSAndrew Gallatin int idx; 2739b2fc195eSAndrew Gallatin 27401e413cf9SAndrew Gallatin sc = ss->sc; 2741b2fc195eSAndrew Gallatin ifp = sc->ifp; 27421e413cf9SAndrew Gallatin rx = &ss->rx_small; 2743b2fc195eSAndrew Gallatin idx = rx->cnt & rx->mask; 2744b2fc195eSAndrew Gallatin rx->cnt++; 2745b2fc195eSAndrew Gallatin /* save a pointer to the received mbuf */ 2746b2fc195eSAndrew Gallatin m = rx->info[idx].m; 2747b2fc195eSAndrew Gallatin /* try to replace the received mbuf */ 27481e413cf9SAndrew Gallatin if (mxge_get_buf_small(ss, rx->extra_map, idx)) { 2749b2fc195eSAndrew Gallatin /* drop the frame -- the old mbuf is re-cycled */ 2750f3f040d9SGleb Smirnoff if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); 2751b2fc195eSAndrew Gallatin return; 2752b2fc195eSAndrew Gallatin } 2753b2fc195eSAndrew Gallatin 2754b2fc195eSAndrew Gallatin /* unmap the received buffer */ 2755b2fc195eSAndrew Gallatin old_map = rx->info[idx].map; 2756b2fc195eSAndrew Gallatin bus_dmamap_sync(rx->dmat, old_map, BUS_DMASYNC_POSTREAD); 2757b2fc195eSAndrew Gallatin bus_dmamap_unload(rx->dmat, old_map); 2758b2fc195eSAndrew Gallatin 2759b2fc195eSAndrew Gallatin /* swap the bus_dmamap_t's */ 2760b2fc195eSAndrew Gallatin rx->info[idx].map = rx->extra_map; 2761b2fc195eSAndrew Gallatin rx->extra_map = old_map; 2762b2fc195eSAndrew Gallatin 2763b2fc195eSAndrew Gallatin /* mcp implicitly skips 1st 2 bytes so that packet is properly 2764b2fc195eSAndrew Gallatin * aligned */ 27655e7d8541SAndrew Gallatin m->m_data += MXGEFW_PAD; 2766b2fc195eSAndrew Gallatin 27679b03b0f3SAndrew Gallatin m->m_pkthdr.rcvif = ifp; 27689b03b0f3SAndrew Gallatin m->m_len = m->m_pkthdr.len = len; 27691e413cf9SAndrew Gallatin ss->ipackets++; 2770c792928fSAndrew Gallatin eh = mtod(m, struct ether_header *); 2771c792928fSAndrew Gallatin if (eh->ether_type == htons(ETHERTYPE_VLAN)) { 2772c792928fSAndrew Gallatin mxge_vlan_tag_remove(m, &csum); 2773c792928fSAndrew Gallatin } 2774eacb70baSSepherosa Ziehau /* flowid only valid if RSS hashing is enabled */ 2775eacb70baSSepherosa Ziehau if (sc->num_slices > 1) { 2776eacb70baSSepherosa Ziehau m->m_pkthdr.flowid = (ss - sc->ss); 2777eacb70baSSepherosa Ziehau M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 2778eacb70baSSepherosa Ziehau } 2779b2fc195eSAndrew Gallatin /* if the checksum is valid, mark it in the mbuf header */ 278026dd49c6SAndrew Gallatin if ((ifp->if_capenable & (IFCAP_RXCSUM_IPV6 | IFCAP_RXCSUM)) && 278126dd49c6SAndrew Gallatin (0 == mxge_rx_csum(m, csum))) { 278226dd49c6SAndrew Gallatin /* Tell the stack that the checksum is good */ 2783053e637fSAndrew Gallatin m->m_pkthdr.csum_data = 0xffff; 278426dd49c6SAndrew Gallatin m->m_pkthdr.csum_flags = CSUM_PSEUDO_HDR | 278526dd49c6SAndrew Gallatin CSUM_DATA_VALID; 278626dd49c6SAndrew Gallatin 278726dd49c6SAndrew Gallatin #if defined(INET) || defined (INET6) 278826dd49c6SAndrew Gallatin if (lro && (0 == tcp_lro_rx(&ss->lc, m, csum))) 278926dd49c6SAndrew Gallatin return; 279026dd49c6SAndrew Gallatin #endif 2791053e637fSAndrew Gallatin } 2792b2fc195eSAndrew Gallatin /* pass the frame up the stack */ 2793b2fc195eSAndrew Gallatin (*ifp->if_input)(ifp, m); 2794b2fc195eSAndrew Gallatin } 2795b2fc195eSAndrew Gallatin 2796b2fc195eSAndrew Gallatin static inline void 27971e413cf9SAndrew Gallatin mxge_clean_rx_done(struct mxge_slice_state *ss) 27985e7d8541SAndrew Gallatin { 27991e413cf9SAndrew Gallatin mxge_rx_done_t *rx_done = &ss->rx_done; 28005e7d8541SAndrew Gallatin int limit = 0; 28015e7d8541SAndrew Gallatin uint16_t length; 28025e7d8541SAndrew Gallatin uint16_t checksum; 280326dd49c6SAndrew Gallatin int lro; 28045e7d8541SAndrew Gallatin 280526dd49c6SAndrew Gallatin lro = ss->sc->ifp->if_capenable & IFCAP_LRO; 28065e7d8541SAndrew Gallatin while (rx_done->entry[rx_done->idx].length != 0) { 28075e7d8541SAndrew Gallatin length = ntohs(rx_done->entry[rx_done->idx].length); 28085e7d8541SAndrew Gallatin rx_done->entry[rx_done->idx].length = 0; 2809053e637fSAndrew Gallatin checksum = rx_done->entry[rx_done->idx].checksum; 2810b4db9009SAndrew Gallatin if (length <= (MHLEN - MXGEFW_PAD)) 281126dd49c6SAndrew Gallatin mxge_rx_done_small(ss, length, checksum, lro); 28125e7d8541SAndrew Gallatin else 281326dd49c6SAndrew Gallatin mxge_rx_done_big(ss, length, checksum, lro); 28145e7d8541SAndrew Gallatin rx_done->cnt++; 2815adae7080SAndrew Gallatin rx_done->idx = rx_done->cnt & rx_done->mask; 28165e7d8541SAndrew Gallatin 28175e7d8541SAndrew Gallatin /* limit potential for livelock */ 2818f616ebc7SAndrew Gallatin if (__predict_false(++limit > rx_done->mask / 2)) 28195e7d8541SAndrew Gallatin break; 2820053e637fSAndrew Gallatin } 282126dd49c6SAndrew Gallatin #if defined(INET) || defined (INET6) 28226dd38b87SSepherosa Ziehau tcp_lro_flush_all(&ss->lc); 2823eb6219e3SAndrew Gallatin #endif 28245e7d8541SAndrew Gallatin } 28255e7d8541SAndrew Gallatin 28265e7d8541SAndrew Gallatin 28275e7d8541SAndrew Gallatin static inline void 28281e413cf9SAndrew Gallatin mxge_tx_done(struct mxge_slice_state *ss, uint32_t mcp_idx) 2829b2fc195eSAndrew Gallatin { 2830b2fc195eSAndrew Gallatin struct ifnet *ifp; 28311e413cf9SAndrew Gallatin mxge_tx_ring_t *tx; 2832b2fc195eSAndrew Gallatin struct mbuf *m; 2833b2fc195eSAndrew Gallatin bus_dmamap_t map; 2834f616ebc7SAndrew Gallatin int idx; 2835c6cb3e3fSAndrew Gallatin int *flags; 2836b2fc195eSAndrew Gallatin 28371e413cf9SAndrew Gallatin tx = &ss->tx; 28381e413cf9SAndrew Gallatin ifp = ss->sc->ifp; 28395e7d8541SAndrew Gallatin while (tx->pkt_done != mcp_idx) { 2840b2fc195eSAndrew Gallatin idx = tx->done & tx->mask; 2841b2fc195eSAndrew Gallatin tx->done++; 2842b2fc195eSAndrew Gallatin m = tx->info[idx].m; 2843b2fc195eSAndrew Gallatin /* mbuf and DMA map only attached to the first 2844b2fc195eSAndrew Gallatin segment per-mbuf */ 2845b2fc195eSAndrew Gallatin if (m != NULL) { 284671032832SAndrew Gallatin ss->obytes += m->m_pkthdr.len; 284771032832SAndrew Gallatin if (m->m_flags & M_MCAST) 284871032832SAndrew Gallatin ss->omcasts++; 2849c6cb3e3fSAndrew Gallatin ss->opackets++; 2850b2fc195eSAndrew Gallatin tx->info[idx].m = NULL; 2851b2fc195eSAndrew Gallatin map = tx->info[idx].map; 2852b2fc195eSAndrew Gallatin bus_dmamap_unload(tx->dmat, map); 2853b2fc195eSAndrew Gallatin m_freem(m); 2854b2fc195eSAndrew Gallatin } 28555e7d8541SAndrew Gallatin if (tx->info[idx].flag) { 28565e7d8541SAndrew Gallatin tx->info[idx].flag = 0; 28575e7d8541SAndrew Gallatin tx->pkt_done++; 28585e7d8541SAndrew Gallatin } 2859b2fc195eSAndrew Gallatin } 2860b2fc195eSAndrew Gallatin 2861b2fc195eSAndrew Gallatin /* If we have space, clear IFF_OACTIVE to tell the stack that 2862b2fc195eSAndrew Gallatin its OK to send packets */ 2863c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING 2864c6cb3e3fSAndrew Gallatin flags = &ss->if_drv_flags; 2865c6cb3e3fSAndrew Gallatin #else 2866c6cb3e3fSAndrew Gallatin flags = &ifp->if_drv_flags; 2867c6cb3e3fSAndrew Gallatin #endif 28681e413cf9SAndrew Gallatin mtx_lock(&ss->tx.mtx); 2869c6cb3e3fSAndrew Gallatin if ((*flags) & IFF_DRV_OACTIVE && 2870c6cb3e3fSAndrew Gallatin tx->req - tx->done < (tx->mask + 1)/4) { 2871c6cb3e3fSAndrew Gallatin *(flags) &= ~IFF_DRV_OACTIVE; 28721e413cf9SAndrew Gallatin ss->tx.wake++; 28731e413cf9SAndrew Gallatin mxge_start_locked(ss); 2874b2fc195eSAndrew Gallatin } 2875c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING 2876c6cb3e3fSAndrew Gallatin if ((ss->sc->num_slices > 1) && (tx->req == tx->done)) { 2877c6cb3e3fSAndrew Gallatin /* let the NIC stop polling this queue, since there 2878c6cb3e3fSAndrew Gallatin * are no more transmits pending */ 2879c6cb3e3fSAndrew Gallatin if (tx->req == tx->done) { 2880c6cb3e3fSAndrew Gallatin *tx->send_stop = 1; 2881c6cb3e3fSAndrew Gallatin tx->queue_active = 0; 2882c6cb3e3fSAndrew Gallatin tx->deactivate++; 2883c6cb3e3fSAndrew Gallatin wmb(); 2884c6cb3e3fSAndrew Gallatin } 2885c6cb3e3fSAndrew Gallatin } 2886c6cb3e3fSAndrew Gallatin #endif 2887c6cb3e3fSAndrew Gallatin mtx_unlock(&ss->tx.mtx); 2888c6cb3e3fSAndrew Gallatin 2889b2fc195eSAndrew Gallatin } 2890b2fc195eSAndrew Gallatin 289101638550SAndrew Gallatin static struct mxge_media_type mxge_xfp_media_types[] = 2892c587e59fSAndrew Gallatin { 2893c587e59fSAndrew Gallatin {IFM_10G_CX4, 0x7f, "10GBASE-CX4 (module)"}, 2894c587e59fSAndrew Gallatin {IFM_10G_SR, (1 << 7), "10GBASE-SR"}, 2895c587e59fSAndrew Gallatin {IFM_10G_LR, (1 << 6), "10GBASE-LR"}, 2896c587e59fSAndrew Gallatin {0, (1 << 5), "10GBASE-ER"}, 289701638550SAndrew Gallatin {IFM_10G_LRM, (1 << 4), "10GBASE-LRM"}, 2898c587e59fSAndrew Gallatin {0, (1 << 3), "10GBASE-SW"}, 2899c587e59fSAndrew Gallatin {0, (1 << 2), "10GBASE-LW"}, 2900c587e59fSAndrew Gallatin {0, (1 << 1), "10GBASE-EW"}, 2901c587e59fSAndrew Gallatin {0, (1 << 0), "Reserved"} 2902c587e59fSAndrew Gallatin }; 290301638550SAndrew Gallatin static struct mxge_media_type mxge_sfp_media_types[] = 290401638550SAndrew Gallatin { 290551bc2092SAndrew Gallatin {IFM_10G_TWINAX, 0, "10GBASE-Twinax"}, 29064ae3322fSAndrew Gallatin {0, (1 << 7), "Reserved"}, 290701638550SAndrew Gallatin {IFM_10G_LRM, (1 << 6), "10GBASE-LRM"}, 290801638550SAndrew Gallatin {IFM_10G_LR, (1 << 5), "10GBASE-LR"}, 290956b67858SAndrew Gallatin {IFM_10G_SR, (1 << 4), "10GBASE-SR"}, 291056b67858SAndrew Gallatin {IFM_10G_TWINAX,(1 << 0), "10GBASE-Twinax"} 291101638550SAndrew Gallatin }; 2912c587e59fSAndrew Gallatin 2913c587e59fSAndrew Gallatin static void 2914c406ad2eSAndrew Gallatin mxge_media_set(mxge_softc_t *sc, int media_type) 2915c587e59fSAndrew Gallatin { 2916c406ad2eSAndrew Gallatin 2917c406ad2eSAndrew Gallatin 2918c406ad2eSAndrew Gallatin ifmedia_add(&sc->media, IFM_ETHER | IFM_FDX | media_type, 2919c406ad2eSAndrew Gallatin 0, NULL); 2920c406ad2eSAndrew Gallatin ifmedia_set(&sc->media, IFM_ETHER | IFM_FDX | media_type); 2921c406ad2eSAndrew Gallatin sc->current_media = media_type; 2922c406ad2eSAndrew Gallatin sc->media.ifm_media = sc->media.ifm_cur->ifm_media; 2923c587e59fSAndrew Gallatin } 2924c587e59fSAndrew Gallatin 2925c587e59fSAndrew Gallatin static void 2926c406ad2eSAndrew Gallatin mxge_media_init(mxge_softc_t *sc) 2927c587e59fSAndrew Gallatin { 2928c587e59fSAndrew Gallatin char *ptr; 2929c406ad2eSAndrew Gallatin int i; 2930c587e59fSAndrew Gallatin 2931c406ad2eSAndrew Gallatin ifmedia_removeall(&sc->media); 2932c406ad2eSAndrew Gallatin mxge_media_set(sc, IFM_AUTO); 2933c587e59fSAndrew Gallatin 2934c587e59fSAndrew Gallatin /* 2935c587e59fSAndrew Gallatin * parse the product code to deterimine the interface type 2936c587e59fSAndrew Gallatin * (CX4, XFP, Quad Ribbon Fiber) by looking at the character 2937c587e59fSAndrew Gallatin * after the 3rd dash in the driver's cached copy of the 2938c587e59fSAndrew Gallatin * EEPROM's product code string. 2939c587e59fSAndrew Gallatin */ 2940c587e59fSAndrew Gallatin ptr = sc->product_code_string; 2941c587e59fSAndrew Gallatin if (ptr == NULL) { 2942c587e59fSAndrew Gallatin device_printf(sc->dev, "Missing product code\n"); 2943c406ad2eSAndrew Gallatin return; 2944c587e59fSAndrew Gallatin } 2945c587e59fSAndrew Gallatin 2946c587e59fSAndrew Gallatin for (i = 0; i < 3; i++, ptr++) { 2947dc15eac0SEd Schouten ptr = strchr(ptr, '-'); 2948c587e59fSAndrew Gallatin if (ptr == NULL) { 2949c587e59fSAndrew Gallatin device_printf(sc->dev, 2950c587e59fSAndrew Gallatin "only %d dashes in PC?!?\n", i); 2951c587e59fSAndrew Gallatin return; 2952c587e59fSAndrew Gallatin } 2953c587e59fSAndrew Gallatin } 295430882b10SAndrew Gallatin if (*ptr == 'C' || *(ptr +1) == 'C') { 295501638550SAndrew Gallatin /* -C is CX4 */ 2956c406ad2eSAndrew Gallatin sc->connector = MXGE_CX4; 2957c406ad2eSAndrew Gallatin mxge_media_set(sc, IFM_10G_CX4); 2958c406ad2eSAndrew Gallatin } else if (*ptr == 'Q') { 295901638550SAndrew Gallatin /* -Q is Quad Ribbon Fiber */ 2960c406ad2eSAndrew Gallatin sc->connector = MXGE_QRF; 2961c587e59fSAndrew Gallatin device_printf(sc->dev, "Quad Ribbon Fiber Media\n"); 2962c587e59fSAndrew Gallatin /* FreeBSD has no media type for Quad ribbon fiber */ 2963c406ad2eSAndrew Gallatin } else if (*ptr == 'R') { 2964c406ad2eSAndrew Gallatin /* -R is XFP */ 2965c406ad2eSAndrew Gallatin sc->connector = MXGE_XFP; 2966c406ad2eSAndrew Gallatin } else if (*ptr == 'S' || *(ptr +1) == 'S') { 2967c406ad2eSAndrew Gallatin /* -S or -2S is SFP+ */ 2968c406ad2eSAndrew Gallatin sc->connector = MXGE_SFP; 2969c406ad2eSAndrew Gallatin } else { 2970c406ad2eSAndrew Gallatin device_printf(sc->dev, "Unknown media type: %c\n", *ptr); 2971c406ad2eSAndrew Gallatin } 2972c587e59fSAndrew Gallatin } 2973c587e59fSAndrew Gallatin 2974c406ad2eSAndrew Gallatin /* 2975c406ad2eSAndrew Gallatin * Determine the media type for a NIC. Some XFPs will identify 2976c406ad2eSAndrew Gallatin * themselves only when their link is up, so this is initiated via a 2977c406ad2eSAndrew Gallatin * link up interrupt. However, this can potentially take up to 2978c406ad2eSAndrew Gallatin * several milliseconds, so it is run via the watchdog routine, rather 2979c406ad2eSAndrew Gallatin * than in the interrupt handler itself. 2980c406ad2eSAndrew Gallatin */ 2981c406ad2eSAndrew Gallatin static void 2982c406ad2eSAndrew Gallatin mxge_media_probe(mxge_softc_t *sc) 2983c406ad2eSAndrew Gallatin { 2984c406ad2eSAndrew Gallatin mxge_cmd_t cmd; 2985c406ad2eSAndrew Gallatin char *cage_type; 2986c406ad2eSAndrew Gallatin 2987c406ad2eSAndrew Gallatin struct mxge_media_type *mxge_media_types = NULL; 2988c406ad2eSAndrew Gallatin int i, err, ms, mxge_media_type_entries; 2989c406ad2eSAndrew Gallatin uint32_t byte; 2990c406ad2eSAndrew Gallatin 2991c406ad2eSAndrew Gallatin sc->need_media_probe = 0; 2992c406ad2eSAndrew Gallatin 2993c406ad2eSAndrew Gallatin if (sc->connector == MXGE_XFP) { 299401638550SAndrew Gallatin /* -R is XFP */ 299501638550SAndrew Gallatin mxge_media_types = mxge_xfp_media_types; 299601638550SAndrew Gallatin mxge_media_type_entries = 299773a1170aSPedro F. Giffuni nitems(mxge_xfp_media_types); 299801638550SAndrew Gallatin byte = MXGE_XFP_COMPLIANCE_BYTE; 299901638550SAndrew Gallatin cage_type = "XFP"; 3000c406ad2eSAndrew Gallatin } else if (sc->connector == MXGE_SFP) { 300101638550SAndrew Gallatin /* -S or -2S is SFP+ */ 300201638550SAndrew Gallatin mxge_media_types = mxge_sfp_media_types; 300301638550SAndrew Gallatin mxge_media_type_entries = 300473a1170aSPedro F. Giffuni nitems(mxge_sfp_media_types); 300501638550SAndrew Gallatin cage_type = "SFP+"; 300601638550SAndrew Gallatin byte = 3; 3007c406ad2eSAndrew Gallatin } else { 3008c406ad2eSAndrew Gallatin /* nothing to do; media type cannot change */ 3009c587e59fSAndrew Gallatin return; 3010c587e59fSAndrew Gallatin } 3011c587e59fSAndrew Gallatin 3012c587e59fSAndrew Gallatin /* 3013c587e59fSAndrew Gallatin * At this point we know the NIC has an XFP cage, so now we 3014c587e59fSAndrew Gallatin * try to determine what is in the cage by using the 3015c587e59fSAndrew Gallatin * firmware's XFP I2C commands to read the XFP 10GbE compilance 3016c587e59fSAndrew Gallatin * register. We read just one byte, which may take over 3017c587e59fSAndrew Gallatin * a millisecond 3018c587e59fSAndrew Gallatin */ 3019c587e59fSAndrew Gallatin 3020c587e59fSAndrew Gallatin cmd.data0 = 0; /* just fetch 1 byte, not all 256 */ 302101638550SAndrew Gallatin cmd.data1 = byte; 302201638550SAndrew Gallatin err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_READ, &cmd); 302301638550SAndrew Gallatin if (err == MXGEFW_CMD_ERROR_I2C_FAILURE) { 3024c587e59fSAndrew Gallatin device_printf(sc->dev, "failed to read XFP\n"); 3025c587e59fSAndrew Gallatin } 302601638550SAndrew Gallatin if (err == MXGEFW_CMD_ERROR_I2C_ABSENT) { 302701638550SAndrew Gallatin device_printf(sc->dev, "Type R/S with no XFP!?!?\n"); 3028c587e59fSAndrew Gallatin } 3029c587e59fSAndrew Gallatin if (err != MXGEFW_CMD_OK) { 3030c587e59fSAndrew Gallatin return; 3031c587e59fSAndrew Gallatin } 3032c587e59fSAndrew Gallatin 3033c587e59fSAndrew Gallatin /* now we wait for the data to be cached */ 303401638550SAndrew Gallatin cmd.data0 = byte; 303501638550SAndrew Gallatin err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_BYTE, &cmd); 3036c587e59fSAndrew Gallatin for (ms = 0; (err == EBUSY) && (ms < 50); ms++) { 3037c587e59fSAndrew Gallatin DELAY(1000); 303801638550SAndrew Gallatin cmd.data0 = byte; 303901638550SAndrew Gallatin err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_BYTE, &cmd); 3040c587e59fSAndrew Gallatin } 3041c587e59fSAndrew Gallatin if (err != MXGEFW_CMD_OK) { 304201638550SAndrew Gallatin device_printf(sc->dev, "failed to read %s (%d, %dms)\n", 304301638550SAndrew Gallatin cage_type, err, ms); 3044c587e59fSAndrew Gallatin return; 3045c587e59fSAndrew Gallatin } 3046c587e59fSAndrew Gallatin 3047c587e59fSAndrew Gallatin if (cmd.data0 == mxge_media_types[0].bitmask) { 3048c587e59fSAndrew Gallatin if (mxge_verbose) 304901638550SAndrew Gallatin device_printf(sc->dev, "%s:%s\n", cage_type, 3050c587e59fSAndrew Gallatin mxge_media_types[0].name); 3051c406ad2eSAndrew Gallatin if (sc->current_media != mxge_media_types[0].flag) { 3052c406ad2eSAndrew Gallatin mxge_media_init(sc); 3053c406ad2eSAndrew Gallatin mxge_media_set(sc, mxge_media_types[0].flag); 3054c406ad2eSAndrew Gallatin } 3055c587e59fSAndrew Gallatin return; 3056c587e59fSAndrew Gallatin } 305701638550SAndrew Gallatin for (i = 1; i < mxge_media_type_entries; i++) { 3058c587e59fSAndrew Gallatin if (cmd.data0 & mxge_media_types[i].bitmask) { 3059c587e59fSAndrew Gallatin if (mxge_verbose) 306001638550SAndrew Gallatin device_printf(sc->dev, "%s:%s\n", 306101638550SAndrew Gallatin cage_type, 3062c587e59fSAndrew Gallatin mxge_media_types[i].name); 3063c587e59fSAndrew Gallatin 3064c406ad2eSAndrew Gallatin if (sc->current_media != mxge_media_types[i].flag) { 3065c406ad2eSAndrew Gallatin mxge_media_init(sc); 3066c406ad2eSAndrew Gallatin mxge_media_set(sc, mxge_media_types[i].flag); 3067c406ad2eSAndrew Gallatin } 3068c587e59fSAndrew Gallatin return; 3069c587e59fSAndrew Gallatin } 3070c587e59fSAndrew Gallatin } 3071c406ad2eSAndrew Gallatin if (mxge_verbose) 3072c406ad2eSAndrew Gallatin device_printf(sc->dev, "%s media 0x%x unknown\n", 3073c406ad2eSAndrew Gallatin cage_type, cmd.data0); 3074c587e59fSAndrew Gallatin 3075c587e59fSAndrew Gallatin return; 3076c587e59fSAndrew Gallatin } 3077c587e59fSAndrew Gallatin 3078b2fc195eSAndrew Gallatin static void 30796d87a65dSAndrew Gallatin mxge_intr(void *arg) 3080b2fc195eSAndrew Gallatin { 30811e413cf9SAndrew Gallatin struct mxge_slice_state *ss = arg; 30821e413cf9SAndrew Gallatin mxge_softc_t *sc = ss->sc; 30831e413cf9SAndrew Gallatin mcp_irq_data_t *stats = ss->fw_stats; 30841e413cf9SAndrew Gallatin mxge_tx_ring_t *tx = &ss->tx; 30851e413cf9SAndrew Gallatin mxge_rx_done_t *rx_done = &ss->rx_done; 30865e7d8541SAndrew Gallatin uint32_t send_done_count; 30875e7d8541SAndrew Gallatin uint8_t valid; 3088b2fc195eSAndrew Gallatin 3089b2fc195eSAndrew Gallatin 3090c6cb3e3fSAndrew Gallatin #ifndef IFNET_BUF_RING 30911e413cf9SAndrew Gallatin /* an interrupt on a non-zero slice is implicitly valid 30921e413cf9SAndrew Gallatin since MSI-X irqs are not shared */ 30931e413cf9SAndrew Gallatin if (ss != sc->ss) { 30941e413cf9SAndrew Gallatin mxge_clean_rx_done(ss); 30951e413cf9SAndrew Gallatin *ss->irq_claim = be32toh(3); 30961e413cf9SAndrew Gallatin return; 30971e413cf9SAndrew Gallatin } 3098c6cb3e3fSAndrew Gallatin #endif 30991e413cf9SAndrew Gallatin 31005e7d8541SAndrew Gallatin /* make sure the DMA has finished */ 31015e7d8541SAndrew Gallatin if (!stats->valid) { 31025e7d8541SAndrew Gallatin return; 3103b2fc195eSAndrew Gallatin } 31045e7d8541SAndrew Gallatin valid = stats->valid; 3105b2fc195eSAndrew Gallatin 310691ed8913SAndrew Gallatin if (sc->legacy_irq) { 31075e7d8541SAndrew Gallatin /* lower legacy IRQ */ 31085e7d8541SAndrew Gallatin *sc->irq_deassert = 0; 31095e7d8541SAndrew Gallatin if (!mxge_deassert_wait) 31105e7d8541SAndrew Gallatin /* don't wait for conf. that irq is low */ 31115e7d8541SAndrew Gallatin stats->valid = 0; 3112dc8731d4SAndrew Gallatin } else { 3113dc8731d4SAndrew Gallatin stats->valid = 0; 3114dc8731d4SAndrew Gallatin } 3115dc8731d4SAndrew Gallatin 3116dc8731d4SAndrew Gallatin /* loop while waiting for legacy irq deassertion */ 31175e7d8541SAndrew Gallatin do { 31185e7d8541SAndrew Gallatin /* check for transmit completes and receives */ 31195e7d8541SAndrew Gallatin send_done_count = be32toh(stats->send_done_count); 31205e7d8541SAndrew Gallatin while ((send_done_count != tx->pkt_done) || 31215e7d8541SAndrew Gallatin (rx_done->entry[rx_done->idx].length != 0)) { 3122c6cb3e3fSAndrew Gallatin if (send_done_count != tx->pkt_done) 31231e413cf9SAndrew Gallatin mxge_tx_done(ss, (int)send_done_count); 31241e413cf9SAndrew Gallatin mxge_clean_rx_done(ss); 31255e7d8541SAndrew Gallatin send_done_count = be32toh(stats->send_done_count); 3126b2fc195eSAndrew Gallatin } 312791ed8913SAndrew Gallatin if (sc->legacy_irq && mxge_deassert_wait) 312873c7c83fSAndrew Gallatin wmb(); 31295e7d8541SAndrew Gallatin } while (*((volatile uint8_t *) &stats->valid)); 3130b2fc195eSAndrew Gallatin 3131c6cb3e3fSAndrew Gallatin /* fw link & error stats meaningful only on the first slice */ 3132c6cb3e3fSAndrew Gallatin if (__predict_false((ss == sc->ss) && stats->stats_updated)) { 31335e7d8541SAndrew Gallatin if (sc->link_state != stats->link_up) { 31345e7d8541SAndrew Gallatin sc->link_state = stats->link_up; 3135b2fc195eSAndrew Gallatin if (sc->link_state) { 31365e7d8541SAndrew Gallatin if_link_state_change(sc->ifp, LINK_STATE_UP); 31375e7d8541SAndrew Gallatin if (mxge_verbose) 31385e7d8541SAndrew Gallatin device_printf(sc->dev, "link up\n"); 3139b2fc195eSAndrew Gallatin } else { 31405e7d8541SAndrew Gallatin if_link_state_change(sc->ifp, LINK_STATE_DOWN); 31415e7d8541SAndrew Gallatin if (mxge_verbose) 31425e7d8541SAndrew Gallatin device_printf(sc->dev, "link down\n"); 3143b2fc195eSAndrew Gallatin } 3144c587e59fSAndrew Gallatin sc->need_media_probe = 1; 3145b2fc195eSAndrew Gallatin } 3146b2fc195eSAndrew Gallatin if (sc->rdma_tags_available != 31471e413cf9SAndrew Gallatin be32toh(stats->rdma_tags_available)) { 3148b2fc195eSAndrew Gallatin sc->rdma_tags_available = 31491e413cf9SAndrew Gallatin be32toh(stats->rdma_tags_available); 31505e7d8541SAndrew Gallatin device_printf(sc->dev, "RDMA timed out! %d tags " 31515e7d8541SAndrew Gallatin "left\n", sc->rdma_tags_available); 31525e7d8541SAndrew Gallatin } 3153c587e59fSAndrew Gallatin 3154c587e59fSAndrew Gallatin if (stats->link_down) { 31555e7d8541SAndrew Gallatin sc->down_cnt += stats->link_down; 3156c587e59fSAndrew Gallatin sc->link_state = 0; 3157c587e59fSAndrew Gallatin if_link_state_change(sc->ifp, LINK_STATE_DOWN); 3158c587e59fSAndrew Gallatin } 3159b2fc195eSAndrew Gallatin } 3160b2fc195eSAndrew Gallatin 31615e7d8541SAndrew Gallatin /* check to see if we have rx token to pass back */ 31625e7d8541SAndrew Gallatin if (valid & 0x1) 31631e413cf9SAndrew Gallatin *ss->irq_claim = be32toh(3); 31641e413cf9SAndrew Gallatin *(ss->irq_claim + 1) = be32toh(3); 3165b2fc195eSAndrew Gallatin } 3166b2fc195eSAndrew Gallatin 3167b2fc195eSAndrew Gallatin static void 31686d87a65dSAndrew Gallatin mxge_init(void *arg) 3169b2fc195eSAndrew Gallatin { 31703cae7311SAndrew Gallatin mxge_softc_t *sc = arg; 31713cae7311SAndrew Gallatin struct ifnet *ifp = sc->ifp; 31723cae7311SAndrew Gallatin 31733cae7311SAndrew Gallatin 31743cae7311SAndrew Gallatin mtx_lock(&sc->driver_mtx); 31753cae7311SAndrew Gallatin if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 31763cae7311SAndrew Gallatin (void) mxge_open(sc); 31773cae7311SAndrew Gallatin mtx_unlock(&sc->driver_mtx); 3178b2fc195eSAndrew Gallatin } 3179b2fc195eSAndrew Gallatin 3180b2fc195eSAndrew Gallatin 3181b2fc195eSAndrew Gallatin 3182b2fc195eSAndrew Gallatin static void 31831e413cf9SAndrew Gallatin mxge_free_slice_mbufs(struct mxge_slice_state *ss) 31841e413cf9SAndrew Gallatin { 31851e413cf9SAndrew Gallatin int i; 31861e413cf9SAndrew Gallatin 318726dd49c6SAndrew Gallatin #if defined(INET) || defined(INET6) 318826dd49c6SAndrew Gallatin tcp_lro_free(&ss->lc); 318926dd49c6SAndrew Gallatin #endif 31901e413cf9SAndrew Gallatin for (i = 0; i <= ss->rx_big.mask; i++) { 31911e413cf9SAndrew Gallatin if (ss->rx_big.info[i].m == NULL) 31921e413cf9SAndrew Gallatin continue; 31931e413cf9SAndrew Gallatin bus_dmamap_unload(ss->rx_big.dmat, 31941e413cf9SAndrew Gallatin ss->rx_big.info[i].map); 31951e413cf9SAndrew Gallatin m_freem(ss->rx_big.info[i].m); 31961e413cf9SAndrew Gallatin ss->rx_big.info[i].m = NULL; 31971e413cf9SAndrew Gallatin } 31981e413cf9SAndrew Gallatin 31991e413cf9SAndrew Gallatin for (i = 0; i <= ss->rx_small.mask; i++) { 32001e413cf9SAndrew Gallatin if (ss->rx_small.info[i].m == NULL) 32011e413cf9SAndrew Gallatin continue; 32021e413cf9SAndrew Gallatin bus_dmamap_unload(ss->rx_small.dmat, 32031e413cf9SAndrew Gallatin ss->rx_small.info[i].map); 32041e413cf9SAndrew Gallatin m_freem(ss->rx_small.info[i].m); 32051e413cf9SAndrew Gallatin ss->rx_small.info[i].m = NULL; 32061e413cf9SAndrew Gallatin } 32071e413cf9SAndrew Gallatin 32081e413cf9SAndrew Gallatin /* transmit ring used only on the first slice */ 32091e413cf9SAndrew Gallatin if (ss->tx.info == NULL) 32101e413cf9SAndrew Gallatin return; 32111e413cf9SAndrew Gallatin 32121e413cf9SAndrew Gallatin for (i = 0; i <= ss->tx.mask; i++) { 32131e413cf9SAndrew Gallatin ss->tx.info[i].flag = 0; 32141e413cf9SAndrew Gallatin if (ss->tx.info[i].m == NULL) 32151e413cf9SAndrew Gallatin continue; 32161e413cf9SAndrew Gallatin bus_dmamap_unload(ss->tx.dmat, 32171e413cf9SAndrew Gallatin ss->tx.info[i].map); 32181e413cf9SAndrew Gallatin m_freem(ss->tx.info[i].m); 32191e413cf9SAndrew Gallatin ss->tx.info[i].m = NULL; 32201e413cf9SAndrew Gallatin } 32211e413cf9SAndrew Gallatin } 32221e413cf9SAndrew Gallatin 32231e413cf9SAndrew Gallatin static void 32246d87a65dSAndrew Gallatin mxge_free_mbufs(mxge_softc_t *sc) 3225b2fc195eSAndrew Gallatin { 32261e413cf9SAndrew Gallatin int slice; 32271e413cf9SAndrew Gallatin 32281e413cf9SAndrew Gallatin for (slice = 0; slice < sc->num_slices; slice++) 32291e413cf9SAndrew Gallatin mxge_free_slice_mbufs(&sc->ss[slice]); 32301e413cf9SAndrew Gallatin } 32311e413cf9SAndrew Gallatin 32321e413cf9SAndrew Gallatin static void 32331e413cf9SAndrew Gallatin mxge_free_slice_rings(struct mxge_slice_state *ss) 32341e413cf9SAndrew Gallatin { 3235b2fc195eSAndrew Gallatin int i; 3236b2fc195eSAndrew Gallatin 3237b2fc195eSAndrew Gallatin 32381e413cf9SAndrew Gallatin if (ss->rx_done.entry != NULL) 32391e413cf9SAndrew Gallatin mxge_dma_free(&ss->rx_done.dma); 32401e413cf9SAndrew Gallatin ss->rx_done.entry = NULL; 3241b2fc195eSAndrew Gallatin 32421e413cf9SAndrew Gallatin if (ss->tx.req_bytes != NULL) 32431e413cf9SAndrew Gallatin free(ss->tx.req_bytes, M_DEVBUF); 32441e413cf9SAndrew Gallatin ss->tx.req_bytes = NULL; 32451e413cf9SAndrew Gallatin 32461e413cf9SAndrew Gallatin if (ss->tx.seg_list != NULL) 32471e413cf9SAndrew Gallatin free(ss->tx.seg_list, M_DEVBUF); 32481e413cf9SAndrew Gallatin ss->tx.seg_list = NULL; 32491e413cf9SAndrew Gallatin 32501e413cf9SAndrew Gallatin if (ss->rx_small.shadow != NULL) 32511e413cf9SAndrew Gallatin free(ss->rx_small.shadow, M_DEVBUF); 32521e413cf9SAndrew Gallatin ss->rx_small.shadow = NULL; 32531e413cf9SAndrew Gallatin 32541e413cf9SAndrew Gallatin if (ss->rx_big.shadow != NULL) 32551e413cf9SAndrew Gallatin free(ss->rx_big.shadow, M_DEVBUF); 32561e413cf9SAndrew Gallatin ss->rx_big.shadow = NULL; 32571e413cf9SAndrew Gallatin 32581e413cf9SAndrew Gallatin if (ss->tx.info != NULL) { 32591e413cf9SAndrew Gallatin if (ss->tx.dmat != NULL) { 32601e413cf9SAndrew Gallatin for (i = 0; i <= ss->tx.mask; i++) { 32611e413cf9SAndrew Gallatin bus_dmamap_destroy(ss->tx.dmat, 32621e413cf9SAndrew Gallatin ss->tx.info[i].map); 3263b2fc195eSAndrew Gallatin } 32641e413cf9SAndrew Gallatin bus_dma_tag_destroy(ss->tx.dmat); 32651e413cf9SAndrew Gallatin } 32661e413cf9SAndrew Gallatin free(ss->tx.info, M_DEVBUF); 32671e413cf9SAndrew Gallatin } 32681e413cf9SAndrew Gallatin ss->tx.info = NULL; 32691e413cf9SAndrew Gallatin 32701e413cf9SAndrew Gallatin if (ss->rx_small.info != NULL) { 32711e413cf9SAndrew Gallatin if (ss->rx_small.dmat != NULL) { 32721e413cf9SAndrew Gallatin for (i = 0; i <= ss->rx_small.mask; i++) { 32731e413cf9SAndrew Gallatin bus_dmamap_destroy(ss->rx_small.dmat, 32741e413cf9SAndrew Gallatin ss->rx_small.info[i].map); 32751e413cf9SAndrew Gallatin } 32761e413cf9SAndrew Gallatin bus_dmamap_destroy(ss->rx_small.dmat, 32771e413cf9SAndrew Gallatin ss->rx_small.extra_map); 32781e413cf9SAndrew Gallatin bus_dma_tag_destroy(ss->rx_small.dmat); 32791e413cf9SAndrew Gallatin } 32801e413cf9SAndrew Gallatin free(ss->rx_small.info, M_DEVBUF); 32811e413cf9SAndrew Gallatin } 32821e413cf9SAndrew Gallatin ss->rx_small.info = NULL; 32831e413cf9SAndrew Gallatin 32841e413cf9SAndrew Gallatin if (ss->rx_big.info != NULL) { 32851e413cf9SAndrew Gallatin if (ss->rx_big.dmat != NULL) { 32861e413cf9SAndrew Gallatin for (i = 0; i <= ss->rx_big.mask; i++) { 32871e413cf9SAndrew Gallatin bus_dmamap_destroy(ss->rx_big.dmat, 32881e413cf9SAndrew Gallatin ss->rx_big.info[i].map); 32891e413cf9SAndrew Gallatin } 32901e413cf9SAndrew Gallatin bus_dmamap_destroy(ss->rx_big.dmat, 32911e413cf9SAndrew Gallatin ss->rx_big.extra_map); 32921e413cf9SAndrew Gallatin bus_dma_tag_destroy(ss->rx_big.dmat); 32931e413cf9SAndrew Gallatin } 32941e413cf9SAndrew Gallatin free(ss->rx_big.info, M_DEVBUF); 32951e413cf9SAndrew Gallatin } 32961e413cf9SAndrew Gallatin ss->rx_big.info = NULL; 3297b2fc195eSAndrew Gallatin } 3298b2fc195eSAndrew Gallatin 3299b2fc195eSAndrew Gallatin static void 33006d87a65dSAndrew Gallatin mxge_free_rings(mxge_softc_t *sc) 3301b2fc195eSAndrew Gallatin { 33021e413cf9SAndrew Gallatin int slice; 3303b2fc195eSAndrew Gallatin 33041e413cf9SAndrew Gallatin for (slice = 0; slice < sc->num_slices; slice++) 33051e413cf9SAndrew Gallatin mxge_free_slice_rings(&sc->ss[slice]); 3306c2657176SAndrew Gallatin } 3307b2fc195eSAndrew Gallatin 3308b2fc195eSAndrew Gallatin static int 33091e413cf9SAndrew Gallatin mxge_alloc_slice_rings(struct mxge_slice_state *ss, int rx_ring_entries, 33101e413cf9SAndrew Gallatin int tx_ring_entries) 3311b2fc195eSAndrew Gallatin { 33121e413cf9SAndrew Gallatin mxge_softc_t *sc = ss->sc; 33131e413cf9SAndrew Gallatin size_t bytes; 33141e413cf9SAndrew Gallatin int err, i; 3315b2fc195eSAndrew Gallatin 33161e413cf9SAndrew Gallatin /* allocate per-slice receive resources */ 3317adae7080SAndrew Gallatin 33181e413cf9SAndrew Gallatin ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1; 33191e413cf9SAndrew Gallatin ss->rx_done.mask = (2 * rx_ring_entries) - 1; 3320aed8e389SAndrew Gallatin 3321b2fc195eSAndrew Gallatin /* allocate the rx shadow rings */ 33221e413cf9SAndrew Gallatin bytes = rx_ring_entries * sizeof (*ss->rx_small.shadow); 33231e413cf9SAndrew Gallatin ss->rx_small.shadow = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK); 3324b2fc195eSAndrew Gallatin 33251e413cf9SAndrew Gallatin bytes = rx_ring_entries * sizeof (*ss->rx_big.shadow); 33261e413cf9SAndrew Gallatin ss->rx_big.shadow = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK); 3327b2fc195eSAndrew Gallatin 33281e413cf9SAndrew Gallatin /* allocate the rx host info rings */ 33291e413cf9SAndrew Gallatin bytes = rx_ring_entries * sizeof (*ss->rx_small.info); 33301e413cf9SAndrew Gallatin ss->rx_small.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK); 3331b2fc195eSAndrew Gallatin 33321e413cf9SAndrew Gallatin bytes = rx_ring_entries * sizeof (*ss->rx_big.info); 33331e413cf9SAndrew Gallatin ss->rx_big.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK); 3334b2fc195eSAndrew Gallatin 33351e413cf9SAndrew Gallatin /* allocate the rx busdma resources */ 3336b2fc195eSAndrew Gallatin err = bus_dma_tag_create(sc->parent_dmat, /* parent */ 3337b2fc195eSAndrew Gallatin 1, /* alignment */ 3338b2fc195eSAndrew Gallatin 4096, /* boundary */ 3339b2fc195eSAndrew Gallatin BUS_SPACE_MAXADDR, /* low */ 3340b2fc195eSAndrew Gallatin BUS_SPACE_MAXADDR, /* high */ 3341b2fc195eSAndrew Gallatin NULL, NULL, /* filter */ 3342b2fc195eSAndrew Gallatin MHLEN, /* maxsize */ 3343b2fc195eSAndrew Gallatin 1, /* num segs */ 3344b2fc195eSAndrew Gallatin MHLEN, /* maxsegsize */ 3345b2fc195eSAndrew Gallatin BUS_DMA_ALLOCNOW, /* flags */ 3346b2fc195eSAndrew Gallatin NULL, NULL, /* lock */ 33471e413cf9SAndrew Gallatin &ss->rx_small.dmat); /* tag */ 3348b2fc195eSAndrew Gallatin if (err != 0) { 3349b2fc195eSAndrew Gallatin device_printf(sc->dev, "Err %d allocating rx_small dmat\n", 3350b2fc195eSAndrew Gallatin err); 3351c2ede4b3SMartin Blapp return err; 3352b2fc195eSAndrew Gallatin } 3353b2fc195eSAndrew Gallatin 3354b2fc195eSAndrew Gallatin err = bus_dma_tag_create(sc->parent_dmat, /* parent */ 3355b2fc195eSAndrew Gallatin 1, /* alignment */ 3356b0f7b922SAndrew Gallatin #if MXGE_VIRT_JUMBOS 3357b2fc195eSAndrew Gallatin 4096, /* boundary */ 3358b0f7b922SAndrew Gallatin #else 3359b0f7b922SAndrew Gallatin 0, /* boundary */ 3360b0f7b922SAndrew Gallatin #endif 3361b2fc195eSAndrew Gallatin BUS_SPACE_MAXADDR, /* low */ 3362b2fc195eSAndrew Gallatin BUS_SPACE_MAXADDR, /* high */ 3363b2fc195eSAndrew Gallatin NULL, NULL, /* filter */ 3364053e637fSAndrew Gallatin 3*4096, /* maxsize */ 3365b0f7b922SAndrew Gallatin #if MXGE_VIRT_JUMBOS 3366053e637fSAndrew Gallatin 3, /* num segs */ 3367b2fc195eSAndrew Gallatin 4096, /* maxsegsize*/ 3368b0f7b922SAndrew Gallatin #else 3369b0f7b922SAndrew Gallatin 1, /* num segs */ 3370b0f7b922SAndrew Gallatin MJUM9BYTES, /* maxsegsize*/ 3371b0f7b922SAndrew Gallatin #endif 3372b2fc195eSAndrew Gallatin BUS_DMA_ALLOCNOW, /* flags */ 3373b2fc195eSAndrew Gallatin NULL, NULL, /* lock */ 33741e413cf9SAndrew Gallatin &ss->rx_big.dmat); /* tag */ 3375b2fc195eSAndrew Gallatin if (err != 0) { 3376b2fc195eSAndrew Gallatin device_printf(sc->dev, "Err %d allocating rx_big dmat\n", 3377b2fc195eSAndrew Gallatin err); 3378c2ede4b3SMartin Blapp return err; 3379b2fc195eSAndrew Gallatin } 33801e413cf9SAndrew Gallatin for (i = 0; i <= ss->rx_small.mask; i++) { 33811e413cf9SAndrew Gallatin err = bus_dmamap_create(ss->rx_small.dmat, 0, 33821e413cf9SAndrew Gallatin &ss->rx_small.info[i].map); 3383b2fc195eSAndrew Gallatin if (err != 0) { 3384b2fc195eSAndrew Gallatin device_printf(sc->dev, "Err %d rx_small dmamap\n", 3385b2fc195eSAndrew Gallatin err); 3386c2ede4b3SMartin Blapp return err; 3387b2fc195eSAndrew Gallatin } 3388b2fc195eSAndrew Gallatin } 33891e413cf9SAndrew Gallatin err = bus_dmamap_create(ss->rx_small.dmat, 0, 33901e413cf9SAndrew Gallatin &ss->rx_small.extra_map); 3391b2fc195eSAndrew Gallatin if (err != 0) { 3392b2fc195eSAndrew Gallatin device_printf(sc->dev, "Err %d extra rx_small dmamap\n", 3393b2fc195eSAndrew Gallatin err); 3394c2ede4b3SMartin Blapp return err; 3395b2fc195eSAndrew Gallatin } 3396b2fc195eSAndrew Gallatin 33971e413cf9SAndrew Gallatin for (i = 0; i <= ss->rx_big.mask; i++) { 33981e413cf9SAndrew Gallatin err = bus_dmamap_create(ss->rx_big.dmat, 0, 33991e413cf9SAndrew Gallatin &ss->rx_big.info[i].map); 3400b2fc195eSAndrew Gallatin if (err != 0) { 3401b2fc195eSAndrew Gallatin device_printf(sc->dev, "Err %d rx_big dmamap\n", 3402b2fc195eSAndrew Gallatin err); 3403c2ede4b3SMartin Blapp return err; 3404b2fc195eSAndrew Gallatin } 3405b2fc195eSAndrew Gallatin } 34061e413cf9SAndrew Gallatin err = bus_dmamap_create(ss->rx_big.dmat, 0, 34071e413cf9SAndrew Gallatin &ss->rx_big.extra_map); 3408b2fc195eSAndrew Gallatin if (err != 0) { 3409b2fc195eSAndrew Gallatin device_printf(sc->dev, "Err %d extra rx_big dmamap\n", 3410b2fc195eSAndrew Gallatin err); 3411c2ede4b3SMartin Blapp return err; 34121e413cf9SAndrew Gallatin } 34131e413cf9SAndrew Gallatin 3414b78540b1SGabor Kovesdan /* now allocate TX resources */ 34151e413cf9SAndrew Gallatin 3416c6cb3e3fSAndrew Gallatin #ifndef IFNET_BUF_RING 34171e413cf9SAndrew Gallatin /* only use a single TX ring for now */ 34181e413cf9SAndrew Gallatin if (ss != ss->sc->ss) 34191e413cf9SAndrew Gallatin return 0; 3420c6cb3e3fSAndrew Gallatin #endif 34211e413cf9SAndrew Gallatin 34221e413cf9SAndrew Gallatin ss->tx.mask = tx_ring_entries - 1; 34231e413cf9SAndrew Gallatin ss->tx.max_desc = MIN(MXGE_MAX_SEND_DESC, tx_ring_entries / 4); 34241e413cf9SAndrew Gallatin 34251e413cf9SAndrew Gallatin 34261e413cf9SAndrew Gallatin /* allocate the tx request copy block */ 34271e413cf9SAndrew Gallatin bytes = 8 + 34281e413cf9SAndrew Gallatin sizeof (*ss->tx.req_list) * (ss->tx.max_desc + 4); 34291e413cf9SAndrew Gallatin ss->tx.req_bytes = malloc(bytes, M_DEVBUF, M_WAITOK); 34301e413cf9SAndrew Gallatin /* ensure req_list entries are aligned to 8 bytes */ 34311e413cf9SAndrew Gallatin ss->tx.req_list = (mcp_kreq_ether_send_t *) 34321e413cf9SAndrew Gallatin ((unsigned long)(ss->tx.req_bytes + 7) & ~7UL); 34331e413cf9SAndrew Gallatin 34341e413cf9SAndrew Gallatin /* allocate the tx busdma segment list */ 34351e413cf9SAndrew Gallatin bytes = sizeof (*ss->tx.seg_list) * ss->tx.max_desc; 34361e413cf9SAndrew Gallatin ss->tx.seg_list = (bus_dma_segment_t *) 34371e413cf9SAndrew Gallatin malloc(bytes, M_DEVBUF, M_WAITOK); 34381e413cf9SAndrew Gallatin 34391e413cf9SAndrew Gallatin /* allocate the tx host info ring */ 34401e413cf9SAndrew Gallatin bytes = tx_ring_entries * sizeof (*ss->tx.info); 34411e413cf9SAndrew Gallatin ss->tx.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK); 34421e413cf9SAndrew Gallatin 34431e413cf9SAndrew Gallatin /* allocate the tx busdma resources */ 34441e413cf9SAndrew Gallatin err = bus_dma_tag_create(sc->parent_dmat, /* parent */ 34451e413cf9SAndrew Gallatin 1, /* alignment */ 34461e413cf9SAndrew Gallatin sc->tx_boundary, /* boundary */ 34471e413cf9SAndrew Gallatin BUS_SPACE_MAXADDR, /* low */ 34481e413cf9SAndrew Gallatin BUS_SPACE_MAXADDR, /* high */ 34491e413cf9SAndrew Gallatin NULL, NULL, /* filter */ 34501e413cf9SAndrew Gallatin 65536 + 256, /* maxsize */ 34511e413cf9SAndrew Gallatin ss->tx.max_desc - 2, /* num segs */ 34521e413cf9SAndrew Gallatin sc->tx_boundary, /* maxsegsz */ 34531e413cf9SAndrew Gallatin BUS_DMA_ALLOCNOW, /* flags */ 34541e413cf9SAndrew Gallatin NULL, NULL, /* lock */ 34551e413cf9SAndrew Gallatin &ss->tx.dmat); /* tag */ 34561e413cf9SAndrew Gallatin 34571e413cf9SAndrew Gallatin if (err != 0) { 34581e413cf9SAndrew Gallatin device_printf(sc->dev, "Err %d allocating tx dmat\n", 34591e413cf9SAndrew Gallatin err); 3460c2ede4b3SMartin Blapp return err; 34611e413cf9SAndrew Gallatin } 34621e413cf9SAndrew Gallatin 34631e413cf9SAndrew Gallatin /* now use these tags to setup dmamaps for each slot 34641e413cf9SAndrew Gallatin in the ring */ 34651e413cf9SAndrew Gallatin for (i = 0; i <= ss->tx.mask; i++) { 34661e413cf9SAndrew Gallatin err = bus_dmamap_create(ss->tx.dmat, 0, 34671e413cf9SAndrew Gallatin &ss->tx.info[i].map); 34681e413cf9SAndrew Gallatin if (err != 0) { 34691e413cf9SAndrew Gallatin device_printf(sc->dev, "Err %d tx dmamap\n", 34701e413cf9SAndrew Gallatin err); 3471c2ede4b3SMartin Blapp return err; 34721e413cf9SAndrew Gallatin } 3473b2fc195eSAndrew Gallatin } 3474b2fc195eSAndrew Gallatin return 0; 3475b2fc195eSAndrew Gallatin 3476b2fc195eSAndrew Gallatin } 3477b2fc195eSAndrew Gallatin 34781e413cf9SAndrew Gallatin static int 34791e413cf9SAndrew Gallatin mxge_alloc_rings(mxge_softc_t *sc) 34801e413cf9SAndrew Gallatin { 34811e413cf9SAndrew Gallatin mxge_cmd_t cmd; 34821e413cf9SAndrew Gallatin int tx_ring_size; 34831e413cf9SAndrew Gallatin int tx_ring_entries, rx_ring_entries; 34841e413cf9SAndrew Gallatin int err, slice; 34851e413cf9SAndrew Gallatin 34861e413cf9SAndrew Gallatin /* get ring sizes */ 34871e413cf9SAndrew Gallatin err = mxge_send_cmd(sc, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd); 34881e413cf9SAndrew Gallatin tx_ring_size = cmd.data0; 34891e413cf9SAndrew Gallatin if (err != 0) { 34901e413cf9SAndrew Gallatin device_printf(sc->dev, "Cannot determine tx ring sizes\n"); 34911e413cf9SAndrew Gallatin goto abort; 34921e413cf9SAndrew Gallatin } 34931e413cf9SAndrew Gallatin 34941e413cf9SAndrew Gallatin tx_ring_entries = tx_ring_size / sizeof (mcp_kreq_ether_send_t); 34951e413cf9SAndrew Gallatin rx_ring_entries = sc->rx_ring_size / sizeof (mcp_dma_addr_t); 34961e413cf9SAndrew Gallatin IFQ_SET_MAXLEN(&sc->ifp->if_snd, tx_ring_entries - 1); 34971e413cf9SAndrew Gallatin sc->ifp->if_snd.ifq_drv_maxlen = sc->ifp->if_snd.ifq_maxlen; 34981e413cf9SAndrew Gallatin IFQ_SET_READY(&sc->ifp->if_snd); 34991e413cf9SAndrew Gallatin 35001e413cf9SAndrew Gallatin for (slice = 0; slice < sc->num_slices; slice++) { 35011e413cf9SAndrew Gallatin err = mxge_alloc_slice_rings(&sc->ss[slice], 35021e413cf9SAndrew Gallatin rx_ring_entries, 35031e413cf9SAndrew Gallatin tx_ring_entries); 35041e413cf9SAndrew Gallatin if (err != 0) 35051e413cf9SAndrew Gallatin goto abort; 35061e413cf9SAndrew Gallatin } 35071e413cf9SAndrew Gallatin return 0; 35081e413cf9SAndrew Gallatin 35091e413cf9SAndrew Gallatin abort: 35101e413cf9SAndrew Gallatin mxge_free_rings(sc); 35111e413cf9SAndrew Gallatin return err; 35121e413cf9SAndrew Gallatin 35131e413cf9SAndrew Gallatin } 35141e413cf9SAndrew Gallatin 35151e413cf9SAndrew Gallatin 3516053e637fSAndrew Gallatin static void 3517053e637fSAndrew Gallatin mxge_choose_params(int mtu, int *big_buf_size, int *cl_size, int *nbufs) 3518053e637fSAndrew Gallatin { 3519c792928fSAndrew Gallatin int bufsize = mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + MXGEFW_PAD; 3520053e637fSAndrew Gallatin 3521053e637fSAndrew Gallatin if (bufsize < MCLBYTES) { 3522053e637fSAndrew Gallatin /* easy, everything fits in a single buffer */ 3523053e637fSAndrew Gallatin *big_buf_size = MCLBYTES; 3524053e637fSAndrew Gallatin *cl_size = MCLBYTES; 3525053e637fSAndrew Gallatin *nbufs = 1; 3526053e637fSAndrew Gallatin return; 3527053e637fSAndrew Gallatin } 3528053e637fSAndrew Gallatin 3529053e637fSAndrew Gallatin if (bufsize < MJUMPAGESIZE) { 3530053e637fSAndrew Gallatin /* still easy, everything still fits in a single buffer */ 3531053e637fSAndrew Gallatin *big_buf_size = MJUMPAGESIZE; 3532053e637fSAndrew Gallatin *cl_size = MJUMPAGESIZE; 3533053e637fSAndrew Gallatin *nbufs = 1; 3534053e637fSAndrew Gallatin return; 3535053e637fSAndrew Gallatin } 3536b0f7b922SAndrew Gallatin #if MXGE_VIRT_JUMBOS 3537053e637fSAndrew Gallatin /* now we need to use virtually contiguous buffers */ 3538053e637fSAndrew Gallatin *cl_size = MJUM9BYTES; 3539053e637fSAndrew Gallatin *big_buf_size = 4096; 3540053e637fSAndrew Gallatin *nbufs = mtu / 4096 + 1; 3541053e637fSAndrew Gallatin /* needs to be a power of two, so round up */ 3542053e637fSAndrew Gallatin if (*nbufs == 3) 3543053e637fSAndrew Gallatin *nbufs = 4; 3544b0f7b922SAndrew Gallatin #else 3545b0f7b922SAndrew Gallatin *cl_size = MJUM9BYTES; 3546b0f7b922SAndrew Gallatin *big_buf_size = MJUM9BYTES; 3547b0f7b922SAndrew Gallatin *nbufs = 1; 3548b0f7b922SAndrew Gallatin #endif 3549053e637fSAndrew Gallatin } 3550053e637fSAndrew Gallatin 3551b2fc195eSAndrew Gallatin static int 35521e413cf9SAndrew Gallatin mxge_slice_open(struct mxge_slice_state *ss, int nbufs, int cl_size) 3553b2fc195eSAndrew Gallatin { 35541e413cf9SAndrew Gallatin mxge_softc_t *sc; 35556d87a65dSAndrew Gallatin mxge_cmd_t cmd; 3556b2fc195eSAndrew Gallatin bus_dmamap_t map; 35571e413cf9SAndrew Gallatin int err, i, slice; 3558b2fc195eSAndrew Gallatin 35591e413cf9SAndrew Gallatin 35601e413cf9SAndrew Gallatin sc = ss->sc; 35611e413cf9SAndrew Gallatin slice = ss - sc->ss; 35621e413cf9SAndrew Gallatin 356326dd49c6SAndrew Gallatin #if defined(INET) || defined(INET6) 356426dd49c6SAndrew Gallatin (void)tcp_lro_init(&ss->lc); 356526dd49c6SAndrew Gallatin #endif 356626dd49c6SAndrew Gallatin ss->lc.ifp = sc->ifp; 3567053e637fSAndrew Gallatin 35681e413cf9SAndrew Gallatin /* get the lanai pointers to the send and receive rings */ 35691e413cf9SAndrew Gallatin 35701e413cf9SAndrew Gallatin err = 0; 3571c6cb3e3fSAndrew Gallatin #ifndef IFNET_BUF_RING 35721e413cf9SAndrew Gallatin /* We currently only send from the first slice */ 35731e413cf9SAndrew Gallatin if (slice == 0) { 3574c6cb3e3fSAndrew Gallatin #endif 35751e413cf9SAndrew Gallatin cmd.data0 = slice; 35761e413cf9SAndrew Gallatin err = mxge_send_cmd(sc, MXGEFW_CMD_GET_SEND_OFFSET, &cmd); 35771e413cf9SAndrew Gallatin ss->tx.lanai = 35781e413cf9SAndrew Gallatin (volatile mcp_kreq_ether_send_t *)(sc->sram + cmd.data0); 3579c6cb3e3fSAndrew Gallatin ss->tx.send_go = (volatile uint32_t *) 3580c6cb3e3fSAndrew Gallatin (sc->sram + MXGEFW_ETH_SEND_GO + 64 * slice); 3581c6cb3e3fSAndrew Gallatin ss->tx.send_stop = (volatile uint32_t *) 3582c6cb3e3fSAndrew Gallatin (sc->sram + MXGEFW_ETH_SEND_STOP + 64 * slice); 3583c6cb3e3fSAndrew Gallatin #ifndef IFNET_BUF_RING 35841e413cf9SAndrew Gallatin } 3585c6cb3e3fSAndrew Gallatin #endif 35861e413cf9SAndrew Gallatin cmd.data0 = slice; 35871e413cf9SAndrew Gallatin err |= mxge_send_cmd(sc, 35881e413cf9SAndrew Gallatin MXGEFW_CMD_GET_SMALL_RX_OFFSET, &cmd); 35891e413cf9SAndrew Gallatin ss->rx_small.lanai = 35901e413cf9SAndrew Gallatin (volatile mcp_kreq_ether_recv_t *)(sc->sram + cmd.data0); 35911e413cf9SAndrew Gallatin cmd.data0 = slice; 35921e413cf9SAndrew Gallatin err |= mxge_send_cmd(sc, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd); 35931e413cf9SAndrew Gallatin ss->rx_big.lanai = 35941e413cf9SAndrew Gallatin (volatile mcp_kreq_ether_recv_t *)(sc->sram + cmd.data0); 35951e413cf9SAndrew Gallatin 35961e413cf9SAndrew Gallatin if (err != 0) { 35971e413cf9SAndrew Gallatin device_printf(sc->dev, 35981e413cf9SAndrew Gallatin "failed to get ring sizes or locations\n"); 35991e413cf9SAndrew Gallatin return EIO; 36001e413cf9SAndrew Gallatin } 36011e413cf9SAndrew Gallatin 36021e413cf9SAndrew Gallatin /* stock receive rings */ 36031e413cf9SAndrew Gallatin for (i = 0; i <= ss->rx_small.mask; i++) { 36041e413cf9SAndrew Gallatin map = ss->rx_small.info[i].map; 36051e413cf9SAndrew Gallatin err = mxge_get_buf_small(ss, map, i); 36061e413cf9SAndrew Gallatin if (err) { 36071e413cf9SAndrew Gallatin device_printf(sc->dev, "alloced %d/%d smalls\n", 36081e413cf9SAndrew Gallatin i, ss->rx_small.mask + 1); 36091e413cf9SAndrew Gallatin return ENOMEM; 36101e413cf9SAndrew Gallatin } 36111e413cf9SAndrew Gallatin } 36121e413cf9SAndrew Gallatin for (i = 0; i <= ss->rx_big.mask; i++) { 36131e413cf9SAndrew Gallatin ss->rx_big.shadow[i].addr_low = 0xffffffff; 36141e413cf9SAndrew Gallatin ss->rx_big.shadow[i].addr_high = 0xffffffff; 36151e413cf9SAndrew Gallatin } 36161e413cf9SAndrew Gallatin ss->rx_big.nbufs = nbufs; 36171e413cf9SAndrew Gallatin ss->rx_big.cl_size = cl_size; 36184d9a5852SAndrew Gallatin ss->rx_big.mlen = ss->sc->ifp->if_mtu + ETHER_HDR_LEN + 36194d9a5852SAndrew Gallatin ETHER_VLAN_ENCAP_LEN + MXGEFW_PAD; 36201e413cf9SAndrew Gallatin for (i = 0; i <= ss->rx_big.mask; i += ss->rx_big.nbufs) { 36211e413cf9SAndrew Gallatin map = ss->rx_big.info[i].map; 36221e413cf9SAndrew Gallatin err = mxge_get_buf_big(ss, map, i); 36231e413cf9SAndrew Gallatin if (err) { 36241e413cf9SAndrew Gallatin device_printf(sc->dev, "alloced %d/%d bigs\n", 36251e413cf9SAndrew Gallatin i, ss->rx_big.mask + 1); 36261e413cf9SAndrew Gallatin return ENOMEM; 36271e413cf9SAndrew Gallatin } 36281e413cf9SAndrew Gallatin } 36291e413cf9SAndrew Gallatin return 0; 36301e413cf9SAndrew Gallatin } 36311e413cf9SAndrew Gallatin 36321e413cf9SAndrew Gallatin static int 36331e413cf9SAndrew Gallatin mxge_open(mxge_softc_t *sc) 36341e413cf9SAndrew Gallatin { 36351e413cf9SAndrew Gallatin mxge_cmd_t cmd; 36361e413cf9SAndrew Gallatin int err, big_bytes, nbufs, slice, cl_size, i; 36371e413cf9SAndrew Gallatin bus_addr_t bus; 36381e413cf9SAndrew Gallatin volatile uint8_t *itable; 3639c6cb3e3fSAndrew Gallatin struct mxge_slice_state *ss; 3640b2fc195eSAndrew Gallatin 36417d542e2dSAndrew Gallatin /* Copy the MAC address in case it was overridden */ 36427d542e2dSAndrew Gallatin bcopy(IF_LLADDR(sc->ifp), sc->mac_addr, ETHER_ADDR_LEN); 36437d542e2dSAndrew Gallatin 3644adae7080SAndrew Gallatin err = mxge_reset(sc, 1); 3645b2fc195eSAndrew Gallatin if (err != 0) { 3646b2fc195eSAndrew Gallatin device_printf(sc->dev, "failed to reset\n"); 3647b2fc195eSAndrew Gallatin return EIO; 3648b2fc195eSAndrew Gallatin } 3649b2fc195eSAndrew Gallatin 36501e413cf9SAndrew Gallatin if (sc->num_slices > 1) { 36511e413cf9SAndrew Gallatin /* setup the indirection table */ 36521e413cf9SAndrew Gallatin cmd.data0 = sc->num_slices; 36531e413cf9SAndrew Gallatin err = mxge_send_cmd(sc, MXGEFW_CMD_SET_RSS_TABLE_SIZE, 36541e413cf9SAndrew Gallatin &cmd); 3655b2fc195eSAndrew Gallatin 36561e413cf9SAndrew Gallatin err |= mxge_send_cmd(sc, MXGEFW_CMD_GET_RSS_TABLE_OFFSET, 36571e413cf9SAndrew Gallatin &cmd); 36581e413cf9SAndrew Gallatin if (err != 0) { 36591e413cf9SAndrew Gallatin device_printf(sc->dev, 36601e413cf9SAndrew Gallatin "failed to setup rss tables\n"); 36611e413cf9SAndrew Gallatin return err; 36621e413cf9SAndrew Gallatin } 36631e413cf9SAndrew Gallatin 36641e413cf9SAndrew Gallatin /* just enable an identity mapping */ 36651e413cf9SAndrew Gallatin itable = sc->sram + cmd.data0; 36661e413cf9SAndrew Gallatin for (i = 0; i < sc->num_slices; i++) 36671e413cf9SAndrew Gallatin itable[i] = (uint8_t)i; 36681e413cf9SAndrew Gallatin 36691e413cf9SAndrew Gallatin cmd.data0 = 1; 36701e413cf9SAndrew Gallatin cmd.data1 = mxge_rss_hash_type; 36711e413cf9SAndrew Gallatin err = mxge_send_cmd(sc, MXGEFW_CMD_SET_RSS_ENABLE, &cmd); 36721e413cf9SAndrew Gallatin if (err != 0) { 36731e413cf9SAndrew Gallatin device_printf(sc->dev, "failed to enable slices\n"); 36741e413cf9SAndrew Gallatin return err; 36751e413cf9SAndrew Gallatin } 36761e413cf9SAndrew Gallatin } 36771e413cf9SAndrew Gallatin 36781e413cf9SAndrew Gallatin 36791e413cf9SAndrew Gallatin mxge_choose_params(sc->ifp->if_mtu, &big_bytes, &cl_size, &nbufs); 36801e413cf9SAndrew Gallatin 36811e413cf9SAndrew Gallatin cmd.data0 = nbufs; 3682053e637fSAndrew Gallatin err = mxge_send_cmd(sc, MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS, 3683053e637fSAndrew Gallatin &cmd); 3684053e637fSAndrew Gallatin /* error is only meaningful if we're trying to set 3685053e637fSAndrew Gallatin MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS > 1 */ 36861e413cf9SAndrew Gallatin if (err && nbufs > 1) { 3687053e637fSAndrew Gallatin device_printf(sc->dev, 3688053e637fSAndrew Gallatin "Failed to set alway-use-n to %d\n", 36891e413cf9SAndrew Gallatin nbufs); 3690053e637fSAndrew Gallatin return EIO; 3691053e637fSAndrew Gallatin } 3692b2fc195eSAndrew Gallatin /* Give the firmware the mtu and the big and small buffer 3693b2fc195eSAndrew Gallatin sizes. The firmware wants the big buf size to be a power 3694b2fc195eSAndrew Gallatin of two. Luckily, FreeBSD's clusters are powers of two */ 3695c792928fSAndrew Gallatin cmd.data0 = sc->ifp->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 36965e7d8541SAndrew Gallatin err = mxge_send_cmd(sc, MXGEFW_CMD_SET_MTU, &cmd); 3697b4db9009SAndrew Gallatin cmd.data0 = MHLEN - MXGEFW_PAD; 36985e7d8541SAndrew Gallatin err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE, 3699b2fc195eSAndrew Gallatin &cmd); 3700053e637fSAndrew Gallatin cmd.data0 = big_bytes; 37015e7d8541SAndrew Gallatin err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd); 37020fa7f681SAndrew Gallatin 37030fa7f681SAndrew Gallatin if (err != 0) { 37040fa7f681SAndrew Gallatin device_printf(sc->dev, "failed to setup params\n"); 37050fa7f681SAndrew Gallatin goto abort; 37060fa7f681SAndrew Gallatin } 37070fa7f681SAndrew Gallatin 3708b2fc195eSAndrew Gallatin /* Now give him the pointer to the stats block */ 3709c6cb3e3fSAndrew Gallatin for (slice = 0; 3710c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING 3711c6cb3e3fSAndrew Gallatin slice < sc->num_slices; 3712c6cb3e3fSAndrew Gallatin #else 3713c6cb3e3fSAndrew Gallatin slice < 1; 3714c6cb3e3fSAndrew Gallatin #endif 3715c6cb3e3fSAndrew Gallatin slice++) { 3716c6cb3e3fSAndrew Gallatin ss = &sc->ss[slice]; 3717c6cb3e3fSAndrew Gallatin cmd.data0 = 3718c6cb3e3fSAndrew Gallatin MXGE_LOWPART_TO_U32(ss->fw_stats_dma.bus_addr); 3719c6cb3e3fSAndrew Gallatin cmd.data1 = 3720c6cb3e3fSAndrew Gallatin MXGE_HIGHPART_TO_U32(ss->fw_stats_dma.bus_addr); 37210fa7f681SAndrew Gallatin cmd.data2 = sizeof(struct mcp_irq_data); 3722c6cb3e3fSAndrew Gallatin cmd.data2 |= (slice << 16); 3723c6cb3e3fSAndrew Gallatin err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd); 3724c6cb3e3fSAndrew Gallatin } 37250fa7f681SAndrew Gallatin 37260fa7f681SAndrew Gallatin if (err != 0) { 37271e413cf9SAndrew Gallatin bus = sc->ss->fw_stats_dma.bus_addr; 37280fa7f681SAndrew Gallatin bus += offsetof(struct mcp_irq_data, send_done_count); 37290fa7f681SAndrew Gallatin cmd.data0 = MXGE_LOWPART_TO_U32(bus); 37300fa7f681SAndrew Gallatin cmd.data1 = MXGE_HIGHPART_TO_U32(bus); 37310fa7f681SAndrew Gallatin err = mxge_send_cmd(sc, 37320fa7f681SAndrew Gallatin MXGEFW_CMD_SET_STATS_DMA_OBSOLETE, 37330fa7f681SAndrew Gallatin &cmd); 37340fa7f681SAndrew Gallatin /* Firmware cannot support multicast without STATS_DMA_V2 */ 37350fa7f681SAndrew Gallatin sc->fw_multicast_support = 0; 37360fa7f681SAndrew Gallatin } else { 37370fa7f681SAndrew Gallatin sc->fw_multicast_support = 1; 37380fa7f681SAndrew Gallatin } 3739b2fc195eSAndrew Gallatin 3740b2fc195eSAndrew Gallatin if (err != 0) { 3741b2fc195eSAndrew Gallatin device_printf(sc->dev, "failed to setup params\n"); 3742b2fc195eSAndrew Gallatin goto abort; 3743b2fc195eSAndrew Gallatin } 3744b2fc195eSAndrew Gallatin 37451e413cf9SAndrew Gallatin for (slice = 0; slice < sc->num_slices; slice++) { 37461e413cf9SAndrew Gallatin err = mxge_slice_open(&sc->ss[slice], nbufs, cl_size); 37471e413cf9SAndrew Gallatin if (err != 0) { 37481e413cf9SAndrew Gallatin device_printf(sc->dev, "couldn't open slice %d\n", 37491e413cf9SAndrew Gallatin slice); 37501e413cf9SAndrew Gallatin goto abort; 37511e413cf9SAndrew Gallatin } 37521e413cf9SAndrew Gallatin } 37531e413cf9SAndrew Gallatin 3754b2fc195eSAndrew Gallatin /* Finally, start the firmware running */ 37555e7d8541SAndrew Gallatin err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_UP, &cmd); 3756b2fc195eSAndrew Gallatin if (err) { 3757b2fc195eSAndrew Gallatin device_printf(sc->dev, "Couldn't bring up link\n"); 3758b2fc195eSAndrew Gallatin goto abort; 3759b2fc195eSAndrew Gallatin } 3760c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING 3761c6cb3e3fSAndrew Gallatin for (slice = 0; slice < sc->num_slices; slice++) { 3762c6cb3e3fSAndrew Gallatin ss = &sc->ss[slice]; 3763c6cb3e3fSAndrew Gallatin ss->if_drv_flags |= IFF_DRV_RUNNING; 3764c6cb3e3fSAndrew Gallatin ss->if_drv_flags &= ~IFF_DRV_OACTIVE; 3765c6cb3e3fSAndrew Gallatin } 3766c6cb3e3fSAndrew Gallatin #endif 3767b2fc195eSAndrew Gallatin sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; 3768b2fc195eSAndrew Gallatin sc->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 3769b2fc195eSAndrew Gallatin 3770b2fc195eSAndrew Gallatin return 0; 3771b2fc195eSAndrew Gallatin 3772b2fc195eSAndrew Gallatin 3773b2fc195eSAndrew Gallatin abort: 37746d87a65dSAndrew Gallatin mxge_free_mbufs(sc); 3775a98d6cd7SAndrew Gallatin 3776b2fc195eSAndrew Gallatin return err; 3777b2fc195eSAndrew Gallatin } 3778b2fc195eSAndrew Gallatin 3779b2fc195eSAndrew Gallatin static int 3780a393336bSAndrew Gallatin mxge_close(mxge_softc_t *sc, int down) 3781b2fc195eSAndrew Gallatin { 37826d87a65dSAndrew Gallatin mxge_cmd_t cmd; 3783b2fc195eSAndrew Gallatin int err, old_down_cnt; 3784c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING 3785c6cb3e3fSAndrew Gallatin struct mxge_slice_state *ss; 3786c6cb3e3fSAndrew Gallatin int slice; 3787c6cb3e3fSAndrew Gallatin #endif 3788b2fc195eSAndrew Gallatin 3789c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING 3790c6cb3e3fSAndrew Gallatin for (slice = 0; slice < sc->num_slices; slice++) { 3791c6cb3e3fSAndrew Gallatin ss = &sc->ss[slice]; 3792c6cb3e3fSAndrew Gallatin ss->if_drv_flags &= ~IFF_DRV_RUNNING; 3793c6cb3e3fSAndrew Gallatin } 3794c6cb3e3fSAndrew Gallatin #endif 3795b2fc195eSAndrew Gallatin sc->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3796a393336bSAndrew Gallatin if (!down) { 3797b2fc195eSAndrew Gallatin old_down_cnt = sc->down_cnt; 379873c7c83fSAndrew Gallatin wmb(); 37995e7d8541SAndrew Gallatin err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_DOWN, &cmd); 3800b2fc195eSAndrew Gallatin if (err) { 3801a393336bSAndrew Gallatin device_printf(sc->dev, 3802a393336bSAndrew Gallatin "Couldn't bring down link\n"); 3803b2fc195eSAndrew Gallatin } 3804b2fc195eSAndrew Gallatin if (old_down_cnt == sc->down_cnt) { 3805b2fc195eSAndrew Gallatin /* wait for down irq */ 3806dce01b9bSAndrew Gallatin DELAY(10 * sc->intr_coal_delay); 3807b2fc195eSAndrew Gallatin } 380873c7c83fSAndrew Gallatin wmb(); 3809b2fc195eSAndrew Gallatin if (old_down_cnt == sc->down_cnt) { 3810b2fc195eSAndrew Gallatin device_printf(sc->dev, "never got down irq\n"); 3811b2fc195eSAndrew Gallatin } 3812a393336bSAndrew Gallatin } 38136d87a65dSAndrew Gallatin mxge_free_mbufs(sc); 3814a98d6cd7SAndrew Gallatin 3815b2fc195eSAndrew Gallatin return 0; 3816b2fc195eSAndrew Gallatin } 3817b2fc195eSAndrew Gallatin 3818dce01b9bSAndrew Gallatin static void 3819dce01b9bSAndrew Gallatin mxge_setup_cfg_space(mxge_softc_t *sc) 3820dce01b9bSAndrew Gallatin { 3821dce01b9bSAndrew Gallatin device_t dev = sc->dev; 3822dce01b9bSAndrew Gallatin int reg; 3823c68534f1SScott Long uint16_t lnk, pectl; 3824dce01b9bSAndrew Gallatin 3825dce01b9bSAndrew Gallatin /* find the PCIe link width and set max read request to 4KB*/ 38263b0a4aefSJohn Baldwin if (pci_find_cap(dev, PCIY_EXPRESS, ®) == 0) { 3827dce01b9bSAndrew Gallatin lnk = pci_read_config(dev, reg + 0x12, 2); 3828dce01b9bSAndrew Gallatin sc->link_width = (lnk >> 4) & 0x3f; 3829dce01b9bSAndrew Gallatin 383083d54b59SAndrew Gallatin if (sc->pectl == 0) { 3831dce01b9bSAndrew Gallatin pectl = pci_read_config(dev, reg + 0x8, 2); 3832dce01b9bSAndrew Gallatin pectl = (pectl & ~0x7000) | (5 << 12); 3833dce01b9bSAndrew Gallatin pci_write_config(dev, reg + 0x8, pectl, 2); 383483d54b59SAndrew Gallatin sc->pectl = pectl; 383583d54b59SAndrew Gallatin } else { 383683d54b59SAndrew Gallatin /* restore saved pectl after watchdog reset */ 383783d54b59SAndrew Gallatin pci_write_config(dev, reg + 0x8, sc->pectl, 2); 383883d54b59SAndrew Gallatin } 3839dce01b9bSAndrew Gallatin } 3840dce01b9bSAndrew Gallatin 3841dce01b9bSAndrew Gallatin /* Enable DMA and Memory space access */ 3842dce01b9bSAndrew Gallatin pci_enable_busmaster(dev); 3843dce01b9bSAndrew Gallatin } 3844dce01b9bSAndrew Gallatin 3845dce01b9bSAndrew Gallatin static uint32_t 3846dce01b9bSAndrew Gallatin mxge_read_reboot(mxge_softc_t *sc) 3847dce01b9bSAndrew Gallatin { 3848dce01b9bSAndrew Gallatin device_t dev = sc->dev; 3849dce01b9bSAndrew Gallatin uint32_t vs; 3850dce01b9bSAndrew Gallatin 3851dce01b9bSAndrew Gallatin /* find the vendor specific offset */ 38523b0a4aefSJohn Baldwin if (pci_find_cap(dev, PCIY_VENDOR, &vs) != 0) { 3853dce01b9bSAndrew Gallatin device_printf(sc->dev, 3854dce01b9bSAndrew Gallatin "could not find vendor specific offset\n"); 3855dce01b9bSAndrew Gallatin return (uint32_t)-1; 3856dce01b9bSAndrew Gallatin } 3857dce01b9bSAndrew Gallatin /* enable read32 mode */ 3858dce01b9bSAndrew Gallatin pci_write_config(dev, vs + 0x10, 0x3, 1); 3859dce01b9bSAndrew Gallatin /* tell NIC which register to read */ 3860dce01b9bSAndrew Gallatin pci_write_config(dev, vs + 0x18, 0xfffffff0, 4); 3861dce01b9bSAndrew Gallatin return (pci_read_config(dev, vs + 0x14, 4)); 3862dce01b9bSAndrew Gallatin } 3863dce01b9bSAndrew Gallatin 386472c042dfSAndrew Gallatin static void 386572c042dfSAndrew Gallatin mxge_watchdog_reset(mxge_softc_t *sc) 3866dce01b9bSAndrew Gallatin { 3867e749ef6bSAndrew Gallatin struct pci_devinfo *dinfo; 3868a393336bSAndrew Gallatin struct mxge_slice_state *ss; 3869a393336bSAndrew Gallatin int err, running, s, num_tx_slices = 1; 3870dce01b9bSAndrew Gallatin uint32_t reboot; 3871dce01b9bSAndrew Gallatin uint16_t cmd; 3872dce01b9bSAndrew Gallatin 3873dce01b9bSAndrew Gallatin err = ENXIO; 3874dce01b9bSAndrew Gallatin 3875dce01b9bSAndrew Gallatin device_printf(sc->dev, "Watchdog reset!\n"); 3876dce01b9bSAndrew Gallatin 3877dce01b9bSAndrew Gallatin /* 3878dce01b9bSAndrew Gallatin * check to see if the NIC rebooted. If it did, then all of 3879dce01b9bSAndrew Gallatin * PCI config space has been reset, and things like the 3880dce01b9bSAndrew Gallatin * busmaster bit will be zero. If this is the case, then we 3881dce01b9bSAndrew Gallatin * must restore PCI config space before the NIC can be used 3882dce01b9bSAndrew Gallatin * again 3883dce01b9bSAndrew Gallatin */ 3884dce01b9bSAndrew Gallatin cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2); 3885dce01b9bSAndrew Gallatin if (cmd == 0xffff) { 3886dce01b9bSAndrew Gallatin /* 3887dce01b9bSAndrew Gallatin * maybe the watchdog caught the NIC rebooting; wait 3888dce01b9bSAndrew Gallatin * up to 100ms for it to finish. If it does not come 3889dce01b9bSAndrew Gallatin * back, then give up 3890dce01b9bSAndrew Gallatin */ 3891dce01b9bSAndrew Gallatin DELAY(1000*100); 3892dce01b9bSAndrew Gallatin cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2); 3893dce01b9bSAndrew Gallatin if (cmd == 0xffff) { 3894dce01b9bSAndrew Gallatin device_printf(sc->dev, "NIC disappeared!\n"); 3895dce01b9bSAndrew Gallatin } 3896dce01b9bSAndrew Gallatin } 3897dce01b9bSAndrew Gallatin if ((cmd & PCIM_CMD_BUSMASTEREN) == 0) { 3898dce01b9bSAndrew Gallatin /* print the reboot status */ 3899dce01b9bSAndrew Gallatin reboot = mxge_read_reboot(sc); 3900dce01b9bSAndrew Gallatin device_printf(sc->dev, "NIC rebooted, status = 0x%x\n", 3901dce01b9bSAndrew Gallatin reboot); 3902a393336bSAndrew Gallatin running = sc->ifp->if_drv_flags & IFF_DRV_RUNNING; 3903a393336bSAndrew Gallatin if (running) { 3904a393336bSAndrew Gallatin 3905a393336bSAndrew Gallatin /* 3906a393336bSAndrew Gallatin * quiesce NIC so that TX routines will not try to 3907a393336bSAndrew Gallatin * xmit after restoration of BAR 3908a393336bSAndrew Gallatin */ 3909a393336bSAndrew Gallatin 3910a393336bSAndrew Gallatin /* Mark the link as down */ 3911a393336bSAndrew Gallatin if (sc->link_state) { 3912a393336bSAndrew Gallatin sc->link_state = 0; 3913a393336bSAndrew Gallatin if_link_state_change(sc->ifp, 3914a393336bSAndrew Gallatin LINK_STATE_DOWN); 3915a393336bSAndrew Gallatin } 3916a393336bSAndrew Gallatin #ifdef IFNET_BUF_RING 3917a393336bSAndrew Gallatin num_tx_slices = sc->num_slices; 3918a393336bSAndrew Gallatin #endif 3919a393336bSAndrew Gallatin /* grab all TX locks to ensure no tx */ 3920a393336bSAndrew Gallatin for (s = 0; s < num_tx_slices; s++) { 3921a393336bSAndrew Gallatin ss = &sc->ss[s]; 3922a393336bSAndrew Gallatin mtx_lock(&ss->tx.mtx); 3923a393336bSAndrew Gallatin } 3924a393336bSAndrew Gallatin mxge_close(sc, 1); 3925a393336bSAndrew Gallatin } 3926dce01b9bSAndrew Gallatin /* restore PCI configuration space */ 3927e749ef6bSAndrew Gallatin dinfo = device_get_ivars(sc->dev); 3928e749ef6bSAndrew Gallatin pci_cfg_restore(sc->dev, dinfo); 3929dce01b9bSAndrew Gallatin 3930dce01b9bSAndrew Gallatin /* and redo any changes we made to our config space */ 3931dce01b9bSAndrew Gallatin mxge_setup_cfg_space(sc); 393210882804SAndrew Gallatin 3933a393336bSAndrew Gallatin /* reload f/w */ 3934a393336bSAndrew Gallatin err = mxge_load_firmware(sc, 0); 3935a393336bSAndrew Gallatin if (err) { 3936a393336bSAndrew Gallatin device_printf(sc->dev, 3937a393336bSAndrew Gallatin "Unable to re-load f/w\n"); 393810882804SAndrew Gallatin } 3939a393336bSAndrew Gallatin if (running) { 3940a393336bSAndrew Gallatin if (!err) 3941a393336bSAndrew Gallatin err = mxge_open(sc); 3942a393336bSAndrew Gallatin /* release all TX locks */ 3943a393336bSAndrew Gallatin for (s = 0; s < num_tx_slices; s++) { 3944a393336bSAndrew Gallatin ss = &sc->ss[s]; 394583d54b59SAndrew Gallatin #ifdef IFNET_BUF_RING 394683d54b59SAndrew Gallatin mxge_start_locked(ss); 394783d54b59SAndrew Gallatin #endif 3948a393336bSAndrew Gallatin mtx_unlock(&ss->tx.mtx); 3949a393336bSAndrew Gallatin } 3950a393336bSAndrew Gallatin } 3951a393336bSAndrew Gallatin sc->watchdog_resets++; 3952dce01b9bSAndrew Gallatin } else { 3953c6cb3e3fSAndrew Gallatin device_printf(sc->dev, 395472c042dfSAndrew Gallatin "NIC did not reboot, not resetting\n"); 395572c042dfSAndrew Gallatin err = 0; 395672c042dfSAndrew Gallatin } 395772c042dfSAndrew Gallatin if (err) { 395872c042dfSAndrew Gallatin device_printf(sc->dev, "watchdog reset failed\n"); 395972c042dfSAndrew Gallatin } else { 39606b484a49SAndrew Gallatin if (sc->dying == 2) 39616b484a49SAndrew Gallatin sc->dying = 0; 39626b484a49SAndrew Gallatin callout_reset(&sc->co_hdl, mxge_ticks, mxge_tick, sc); 396372c042dfSAndrew Gallatin } 396472c042dfSAndrew Gallatin } 396572c042dfSAndrew Gallatin 396672c042dfSAndrew Gallatin static void 396772c042dfSAndrew Gallatin mxge_watchdog_task(void *arg, int pending) 396872c042dfSAndrew Gallatin { 396972c042dfSAndrew Gallatin mxge_softc_t *sc = arg; 397072c042dfSAndrew Gallatin 397172c042dfSAndrew Gallatin 397272c042dfSAndrew Gallatin mtx_lock(&sc->driver_mtx); 397372c042dfSAndrew Gallatin mxge_watchdog_reset(sc); 397472c042dfSAndrew Gallatin mtx_unlock(&sc->driver_mtx); 397572c042dfSAndrew Gallatin } 397672c042dfSAndrew Gallatin 397772c042dfSAndrew Gallatin static void 397872c042dfSAndrew Gallatin mxge_warn_stuck(mxge_softc_t *sc, mxge_tx_ring_t *tx, int slice) 397972c042dfSAndrew Gallatin { 398072c042dfSAndrew Gallatin tx = &sc->ss[slice].tx; 398172c042dfSAndrew Gallatin device_printf(sc->dev, "slice %d struck? ring state:\n", slice); 3982c6cb3e3fSAndrew Gallatin device_printf(sc->dev, 3983c6cb3e3fSAndrew Gallatin "tx.req=%d tx.done=%d, tx.queue_active=%d\n", 3984c6cb3e3fSAndrew Gallatin tx->req, tx->done, tx->queue_active); 3985c6cb3e3fSAndrew Gallatin device_printf(sc->dev, "tx.activate=%d tx.deactivate=%d\n", 3986c6cb3e3fSAndrew Gallatin tx->activate, tx->deactivate); 3987dce01b9bSAndrew Gallatin device_printf(sc->dev, "pkt_done=%d fw=%d\n", 3988c6cb3e3fSAndrew Gallatin tx->pkt_done, 39891e413cf9SAndrew Gallatin be32toh(sc->ss->fw_stats->send_done_count)); 3990dce01b9bSAndrew Gallatin } 3991dce01b9bSAndrew Gallatin 3992e749ef6bSAndrew Gallatin static int 3993dce01b9bSAndrew Gallatin mxge_watchdog(mxge_softc_t *sc) 3994dce01b9bSAndrew Gallatin { 3995c6cb3e3fSAndrew Gallatin mxge_tx_ring_t *tx; 39961e413cf9SAndrew Gallatin uint32_t rx_pause = be32toh(sc->ss->fw_stats->dropped_pause); 3997c6cb3e3fSAndrew Gallatin int i, err = 0; 3998dce01b9bSAndrew Gallatin 3999dce01b9bSAndrew Gallatin /* see if we have outstanding transmits, which 4000dce01b9bSAndrew Gallatin have been pending for more than mxge_ticks */ 4001c6cb3e3fSAndrew Gallatin for (i = 0; 4002c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING 4003c6cb3e3fSAndrew Gallatin (i < sc->num_slices) && (err == 0); 4004c6cb3e3fSAndrew Gallatin #else 4005c6cb3e3fSAndrew Gallatin (i < 1) && (err == 0); 4006c6cb3e3fSAndrew Gallatin #endif 4007c6cb3e3fSAndrew Gallatin i++) { 4008c6cb3e3fSAndrew Gallatin tx = &sc->ss[i].tx; 4009dce01b9bSAndrew Gallatin if (tx->req != tx->done && 4010dce01b9bSAndrew Gallatin tx->watchdog_req != tx->watchdog_done && 4011c587e59fSAndrew Gallatin tx->done == tx->watchdog_done) { 4012c587e59fSAndrew Gallatin /* check for pause blocking before resetting */ 401372c042dfSAndrew Gallatin if (tx->watchdog_rx_pause == rx_pause) { 401472c042dfSAndrew Gallatin mxge_warn_stuck(sc, tx, i); 401572c042dfSAndrew Gallatin taskqueue_enqueue(sc->tq, &sc->watchdog_task); 401672c042dfSAndrew Gallatin return (ENXIO); 401772c042dfSAndrew Gallatin } 4018c587e59fSAndrew Gallatin else 4019c587e59fSAndrew Gallatin device_printf(sc->dev, "Flow control blocking " 4020c587e59fSAndrew Gallatin "xmits, check link partner\n"); 4021c587e59fSAndrew Gallatin } 4022dce01b9bSAndrew Gallatin 4023dce01b9bSAndrew Gallatin tx->watchdog_req = tx->req; 4024dce01b9bSAndrew Gallatin tx->watchdog_done = tx->done; 4025c587e59fSAndrew Gallatin tx->watchdog_rx_pause = rx_pause; 4026c6cb3e3fSAndrew Gallatin } 4027c587e59fSAndrew Gallatin 4028c587e59fSAndrew Gallatin if (sc->need_media_probe) 4029c587e59fSAndrew Gallatin mxge_media_probe(sc); 4030e749ef6bSAndrew Gallatin return (err); 4031dce01b9bSAndrew Gallatin } 4032dce01b9bSAndrew Gallatin 4033f3f040d9SGleb Smirnoff static uint64_t 4034f3f040d9SGleb Smirnoff mxge_get_counter(struct ifnet *ifp, ift_counter cnt) 40351e413cf9SAndrew Gallatin { 4036f3f040d9SGleb Smirnoff struct mxge_softc *sc; 4037f3f040d9SGleb Smirnoff uint64_t rv; 40381e413cf9SAndrew Gallatin 4039f3f040d9SGleb Smirnoff sc = if_getsoftc(ifp); 4040f3f040d9SGleb Smirnoff rv = 0; 4041f3f040d9SGleb Smirnoff 4042f3f040d9SGleb Smirnoff switch (cnt) { 4043f3f040d9SGleb Smirnoff case IFCOUNTER_IPACKETS: 4044f3f040d9SGleb Smirnoff for (int s = 0; s < sc->num_slices; s++) 4045f3f040d9SGleb Smirnoff rv += sc->ss[s].ipackets; 4046f3f040d9SGleb Smirnoff return (rv); 4047f3f040d9SGleb Smirnoff case IFCOUNTER_OPACKETS: 4048f3f040d9SGleb Smirnoff for (int s = 0; s < sc->num_slices; s++) 4049f3f040d9SGleb Smirnoff rv += sc->ss[s].opackets; 4050f3f040d9SGleb Smirnoff return (rv); 4051f3f040d9SGleb Smirnoff case IFCOUNTER_OERRORS: 4052f3f040d9SGleb Smirnoff for (int s = 0; s < sc->num_slices; s++) 4053f3f040d9SGleb Smirnoff rv += sc->ss[s].oerrors; 4054f3f040d9SGleb Smirnoff return (rv); 405571032832SAndrew Gallatin #ifdef IFNET_BUF_RING 4056f3f040d9SGleb Smirnoff case IFCOUNTER_OBYTES: 4057f3f040d9SGleb Smirnoff for (int s = 0; s < sc->num_slices; s++) 4058f3f040d9SGleb Smirnoff rv += sc->ss[s].obytes; 4059f3f040d9SGleb Smirnoff return (rv); 4060f3f040d9SGleb Smirnoff case IFCOUNTER_OMCASTS: 4061f3f040d9SGleb Smirnoff for (int s = 0; s < sc->num_slices; s++) 4062f3f040d9SGleb Smirnoff rv += sc->ss[s].omcasts; 4063f3f040d9SGleb Smirnoff return (rv); 4064f3f040d9SGleb Smirnoff case IFCOUNTER_OQDROPS: 4065f3f040d9SGleb Smirnoff for (int s = 0; s < sc->num_slices; s++) 4066f3f040d9SGleb Smirnoff rv += sc->ss[s].tx.br->br_drops; 4067f3f040d9SGleb Smirnoff return (rv); 406871032832SAndrew Gallatin #endif 4069f3f040d9SGleb Smirnoff default: 4070f3f040d9SGleb Smirnoff return (if_get_counter_default(ifp, cnt)); 40711e413cf9SAndrew Gallatin } 40721e413cf9SAndrew Gallatin } 4073c6cb3e3fSAndrew Gallatin 40741e413cf9SAndrew Gallatin static void 4075dce01b9bSAndrew Gallatin mxge_tick(void *arg) 4076dce01b9bSAndrew Gallatin { 4077dce01b9bSAndrew Gallatin mxge_softc_t *sc = arg; 40786b484a49SAndrew Gallatin u_long pkts = 0; 4079e749ef6bSAndrew Gallatin int err = 0; 40806b484a49SAndrew Gallatin int running, ticks; 40816b484a49SAndrew Gallatin uint16_t cmd; 4082dce01b9bSAndrew Gallatin 40836b484a49SAndrew Gallatin ticks = mxge_ticks; 40846b484a49SAndrew Gallatin running = sc->ifp->if_drv_flags & IFF_DRV_RUNNING; 40856b484a49SAndrew Gallatin if (running) { 40861e413cf9SAndrew Gallatin if (!sc->watchdog_countdown) { 4087e749ef6bSAndrew Gallatin err = mxge_watchdog(sc); 40881e413cf9SAndrew Gallatin sc->watchdog_countdown = 4; 40891e413cf9SAndrew Gallatin } 40901e413cf9SAndrew Gallatin sc->watchdog_countdown--; 40916b484a49SAndrew Gallatin } 40926b484a49SAndrew Gallatin if (pkts == 0) { 40936b484a49SAndrew Gallatin /* ensure NIC did not suffer h/w fault while idle */ 40946b484a49SAndrew Gallatin cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2); 40956b484a49SAndrew Gallatin if ((cmd & PCIM_CMD_BUSMASTEREN) == 0) { 40966b484a49SAndrew Gallatin sc->dying = 2; 40976b484a49SAndrew Gallatin taskqueue_enqueue(sc->tq, &sc->watchdog_task); 40986b484a49SAndrew Gallatin err = ENXIO; 40996b484a49SAndrew Gallatin } 41006b484a49SAndrew Gallatin /* look less often if NIC is idle */ 41016b484a49SAndrew Gallatin ticks *= 4; 41026b484a49SAndrew Gallatin } 41036b484a49SAndrew Gallatin 4104e749ef6bSAndrew Gallatin if (err == 0) 41056b484a49SAndrew Gallatin callout_reset(&sc->co_hdl, ticks, mxge_tick, sc); 4106e749ef6bSAndrew Gallatin 4107dce01b9bSAndrew Gallatin } 4108b2fc195eSAndrew Gallatin 4109b2fc195eSAndrew Gallatin static int 41106d87a65dSAndrew Gallatin mxge_media_change(struct ifnet *ifp) 4111b2fc195eSAndrew Gallatin { 4112b2fc195eSAndrew Gallatin return EINVAL; 4113b2fc195eSAndrew Gallatin } 4114b2fc195eSAndrew Gallatin 4115b2fc195eSAndrew Gallatin static int 41166d87a65dSAndrew Gallatin mxge_change_mtu(mxge_softc_t *sc, int mtu) 4117b2fc195eSAndrew Gallatin { 4118b2fc195eSAndrew Gallatin struct ifnet *ifp = sc->ifp; 4119b2fc195eSAndrew Gallatin int real_mtu, old_mtu; 4120b2fc195eSAndrew Gallatin int err = 0; 4121b2fc195eSAndrew Gallatin 4122b2fc195eSAndrew Gallatin 4123c792928fSAndrew Gallatin real_mtu = mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 4124053e637fSAndrew Gallatin if ((real_mtu > sc->max_mtu) || real_mtu < 60) 4125b2fc195eSAndrew Gallatin return EINVAL; 4126a98d6cd7SAndrew Gallatin mtx_lock(&sc->driver_mtx); 4127b2fc195eSAndrew Gallatin old_mtu = ifp->if_mtu; 4128b2fc195eSAndrew Gallatin ifp->if_mtu = mtu; 4129b2fc195eSAndrew Gallatin if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 4130a393336bSAndrew Gallatin mxge_close(sc, 0); 41316d87a65dSAndrew Gallatin err = mxge_open(sc); 4132b2fc195eSAndrew Gallatin if (err != 0) { 4133b2fc195eSAndrew Gallatin ifp->if_mtu = old_mtu; 4134a393336bSAndrew Gallatin mxge_close(sc, 0); 41356d87a65dSAndrew Gallatin (void) mxge_open(sc); 4136b2fc195eSAndrew Gallatin } 4137b2fc195eSAndrew Gallatin } 4138a98d6cd7SAndrew Gallatin mtx_unlock(&sc->driver_mtx); 4139b2fc195eSAndrew Gallatin return err; 4140b2fc195eSAndrew Gallatin } 4141b2fc195eSAndrew Gallatin 4142b2fc195eSAndrew Gallatin static void 41436d87a65dSAndrew Gallatin mxge_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 4144b2fc195eSAndrew Gallatin { 41456d87a65dSAndrew Gallatin mxge_softc_t *sc = ifp->if_softc; 4146b2fc195eSAndrew Gallatin 4147b2fc195eSAndrew Gallatin 4148b2fc195eSAndrew Gallatin if (sc == NULL) 4149b2fc195eSAndrew Gallatin return; 4150b2fc195eSAndrew Gallatin ifmr->ifm_status = IFM_AVALID; 4151c406ad2eSAndrew Gallatin ifmr->ifm_active = IFM_ETHER | IFM_FDX; 4152c587e59fSAndrew Gallatin ifmr->ifm_status |= sc->link_state ? IFM_ACTIVE : 0; 4153c406ad2eSAndrew Gallatin ifmr->ifm_active |= sc->current_media; 4154b2fc195eSAndrew Gallatin } 4155b2fc195eSAndrew Gallatin 4156b2fc195eSAndrew Gallatin static int 41576d87a65dSAndrew Gallatin mxge_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 4158b2fc195eSAndrew Gallatin { 41596d87a65dSAndrew Gallatin mxge_softc_t *sc = ifp->if_softc; 4160b2fc195eSAndrew Gallatin struct ifreq *ifr = (struct ifreq *)data; 4161b2fc195eSAndrew Gallatin int err, mask; 4162b2fc195eSAndrew Gallatin 4163b2fc195eSAndrew Gallatin err = 0; 4164b2fc195eSAndrew Gallatin switch (command) { 4165b2fc195eSAndrew Gallatin case SIOCSIFMTU: 41666d87a65dSAndrew Gallatin err = mxge_change_mtu(sc, ifr->ifr_mtu); 4167b2fc195eSAndrew Gallatin break; 4168b2fc195eSAndrew Gallatin 4169b2fc195eSAndrew Gallatin case SIOCSIFFLAGS: 4170a98d6cd7SAndrew Gallatin mtx_lock(&sc->driver_mtx); 41718c5d766cSAndrew Gallatin if (sc->dying) { 41728c5d766cSAndrew Gallatin mtx_unlock(&sc->driver_mtx); 41738c5d766cSAndrew Gallatin return EINVAL; 41748c5d766cSAndrew Gallatin } 4175b2fc195eSAndrew Gallatin if (ifp->if_flags & IFF_UP) { 4176dce01b9bSAndrew Gallatin if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 41776d87a65dSAndrew Gallatin err = mxge_open(sc); 4178dce01b9bSAndrew Gallatin } else { 41790fa7f681SAndrew Gallatin /* take care of promis can allmulti 41800fa7f681SAndrew Gallatin flag chages */ 41810fa7f681SAndrew Gallatin mxge_change_promisc(sc, 41820fa7f681SAndrew Gallatin ifp->if_flags & IFF_PROMISC); 41830fa7f681SAndrew Gallatin mxge_set_multicast_list(sc); 41840fa7f681SAndrew Gallatin } 4185b2fc195eSAndrew Gallatin } else { 4186dce01b9bSAndrew Gallatin if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 4187a393336bSAndrew Gallatin mxge_close(sc, 0); 4188dce01b9bSAndrew Gallatin } 4189b2fc195eSAndrew Gallatin } 4190a98d6cd7SAndrew Gallatin mtx_unlock(&sc->driver_mtx); 4191b2fc195eSAndrew Gallatin break; 4192b2fc195eSAndrew Gallatin 4193b2fc195eSAndrew Gallatin case SIOCADDMULTI: 4194b2fc195eSAndrew Gallatin case SIOCDELMULTI: 4195a98d6cd7SAndrew Gallatin mtx_lock(&sc->driver_mtx); 41960fa7f681SAndrew Gallatin mxge_set_multicast_list(sc); 4197a98d6cd7SAndrew Gallatin mtx_unlock(&sc->driver_mtx); 4198b2fc195eSAndrew Gallatin break; 4199b2fc195eSAndrew Gallatin 4200b2fc195eSAndrew Gallatin case SIOCSIFCAP: 4201a98d6cd7SAndrew Gallatin mtx_lock(&sc->driver_mtx); 4202b2fc195eSAndrew Gallatin mask = ifr->ifr_reqcap ^ ifp->if_capenable; 4203b2fc195eSAndrew Gallatin if (mask & IFCAP_TXCSUM) { 4204b2fc195eSAndrew Gallatin if (IFCAP_TXCSUM & ifp->if_capenable) { 4205aed8e389SAndrew Gallatin ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4); 42060a7a780eSAndrew Gallatin ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP); 4207b2fc195eSAndrew Gallatin } else { 4208b2fc195eSAndrew Gallatin ifp->if_capenable |= IFCAP_TXCSUM; 4209b2fc195eSAndrew Gallatin ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 4210b2fc195eSAndrew Gallatin } 4211b2fc195eSAndrew Gallatin } else if (mask & IFCAP_RXCSUM) { 4212b2fc195eSAndrew Gallatin if (IFCAP_RXCSUM & ifp->if_capenable) { 4213b2fc195eSAndrew Gallatin ifp->if_capenable &= ~IFCAP_RXCSUM; 4214b2fc195eSAndrew Gallatin } else { 4215b2fc195eSAndrew Gallatin ifp->if_capenable |= IFCAP_RXCSUM; 4216b2fc195eSAndrew Gallatin } 4217b2fc195eSAndrew Gallatin } 4218aed8e389SAndrew Gallatin if (mask & IFCAP_TSO4) { 4219aed8e389SAndrew Gallatin if (IFCAP_TSO4 & ifp->if_capenable) { 4220aed8e389SAndrew Gallatin ifp->if_capenable &= ~IFCAP_TSO4; 4221aed8e389SAndrew Gallatin } else if (IFCAP_TXCSUM & ifp->if_capenable) { 4222aed8e389SAndrew Gallatin ifp->if_capenable |= IFCAP_TSO4; 4223aed8e389SAndrew Gallatin ifp->if_hwassist |= CSUM_TSO; 4224aed8e389SAndrew Gallatin } else { 4225aed8e389SAndrew Gallatin printf("mxge requires tx checksum offload" 4226aed8e389SAndrew Gallatin " be enabled to use TSO\n"); 4227aed8e389SAndrew Gallatin err = EINVAL; 4228aed8e389SAndrew Gallatin } 4229aed8e389SAndrew Gallatin } 42300a7a780eSAndrew Gallatin #if IFCAP_TSO6 42310a7a780eSAndrew Gallatin if (mask & IFCAP_TXCSUM_IPV6) { 42320a7a780eSAndrew Gallatin if (IFCAP_TXCSUM_IPV6 & ifp->if_capenable) { 42330a7a780eSAndrew Gallatin ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 42340a7a780eSAndrew Gallatin | IFCAP_TSO6); 42350a7a780eSAndrew Gallatin ifp->if_hwassist &= ~(CSUM_TCP_IPV6 42360a7a780eSAndrew Gallatin | CSUM_UDP); 42370a7a780eSAndrew Gallatin } else { 42380a7a780eSAndrew Gallatin ifp->if_capenable |= IFCAP_TXCSUM_IPV6; 42390a7a780eSAndrew Gallatin ifp->if_hwassist |= (CSUM_TCP_IPV6 42400a7a780eSAndrew Gallatin | CSUM_UDP_IPV6); 42410a7a780eSAndrew Gallatin } 424226dd49c6SAndrew Gallatin } else if (mask & IFCAP_RXCSUM_IPV6) { 424326dd49c6SAndrew Gallatin if (IFCAP_RXCSUM_IPV6 & ifp->if_capenable) { 424426dd49c6SAndrew Gallatin ifp->if_capenable &= ~IFCAP_RXCSUM_IPV6; 42450a7a780eSAndrew Gallatin } else { 424626dd49c6SAndrew Gallatin ifp->if_capenable |= IFCAP_RXCSUM_IPV6; 42470a7a780eSAndrew Gallatin } 42480a7a780eSAndrew Gallatin } 42490a7a780eSAndrew Gallatin if (mask & IFCAP_TSO6) { 42500a7a780eSAndrew Gallatin if (IFCAP_TSO6 & ifp->if_capenable) { 42510a7a780eSAndrew Gallatin ifp->if_capenable &= ~IFCAP_TSO6; 42520a7a780eSAndrew Gallatin } else if (IFCAP_TXCSUM_IPV6 & ifp->if_capenable) { 42530a7a780eSAndrew Gallatin ifp->if_capenable |= IFCAP_TSO6; 42540a7a780eSAndrew Gallatin ifp->if_hwassist |= CSUM_TSO; 42550a7a780eSAndrew Gallatin } else { 42560a7a780eSAndrew Gallatin printf("mxge requires tx checksum offload" 42570a7a780eSAndrew Gallatin " be enabled to use TSO\n"); 42580a7a780eSAndrew Gallatin err = EINVAL; 42590a7a780eSAndrew Gallatin } 42600a7a780eSAndrew Gallatin } 42610a7a780eSAndrew Gallatin #endif /*IFCAP_TSO6 */ 42620a7a780eSAndrew Gallatin 426326dd49c6SAndrew Gallatin if (mask & IFCAP_LRO) 426426dd49c6SAndrew Gallatin ifp->if_capenable ^= IFCAP_LRO; 4265c792928fSAndrew Gallatin if (mask & IFCAP_VLAN_HWTAGGING) 4266c792928fSAndrew Gallatin ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 42670dce6781SAndrew Gallatin if (mask & IFCAP_VLAN_HWTSO) 42680dce6781SAndrew Gallatin ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 42690dce6781SAndrew Gallatin 42700dce6781SAndrew Gallatin if (!(ifp->if_capabilities & IFCAP_VLAN_HWTSO) || 42710dce6781SAndrew Gallatin !(ifp->if_capenable & IFCAP_VLAN_HWTAGGING)) 42720dce6781SAndrew Gallatin ifp->if_capenable &= ~IFCAP_VLAN_HWTSO; 42730dce6781SAndrew Gallatin 4274a98d6cd7SAndrew Gallatin mtx_unlock(&sc->driver_mtx); 4275c792928fSAndrew Gallatin VLAN_CAPABILITIES(ifp); 4276c792928fSAndrew Gallatin 4277b2fc195eSAndrew Gallatin break; 4278b2fc195eSAndrew Gallatin 4279b2fc195eSAndrew Gallatin case SIOCGIFMEDIA: 4280c406ad2eSAndrew Gallatin mtx_lock(&sc->driver_mtx); 4281c406ad2eSAndrew Gallatin mxge_media_probe(sc); 4282c406ad2eSAndrew Gallatin mtx_unlock(&sc->driver_mtx); 4283b2fc195eSAndrew Gallatin err = ifmedia_ioctl(ifp, (struct ifreq *)data, 4284b2fc195eSAndrew Gallatin &sc->media, command); 4285b2fc195eSAndrew Gallatin break; 4286b2fc195eSAndrew Gallatin 4287b2fc195eSAndrew Gallatin default: 4288*c756fb6eSRavi Pokala err = ether_ioctl(ifp, command, data); 4289*c756fb6eSRavi Pokala break; 4290b2fc195eSAndrew Gallatin } 4291b2fc195eSAndrew Gallatin return err; 4292b2fc195eSAndrew Gallatin } 4293b2fc195eSAndrew Gallatin 4294b2fc195eSAndrew Gallatin static void 42956d87a65dSAndrew Gallatin mxge_fetch_tunables(mxge_softc_t *sc) 4296b2fc195eSAndrew Gallatin { 4297b2fc195eSAndrew Gallatin 42981e413cf9SAndrew Gallatin TUNABLE_INT_FETCH("hw.mxge.max_slices", &mxge_max_slices); 42996d87a65dSAndrew Gallatin TUNABLE_INT_FETCH("hw.mxge.flow_control_enabled", 43006d87a65dSAndrew Gallatin &mxge_flow_control); 43016d87a65dSAndrew Gallatin TUNABLE_INT_FETCH("hw.mxge.intr_coal_delay", 43026d87a65dSAndrew Gallatin &mxge_intr_coal_delay); 43036d87a65dSAndrew Gallatin TUNABLE_INT_FETCH("hw.mxge.nvidia_ecrc_enable", 43046d87a65dSAndrew Gallatin &mxge_nvidia_ecrc_enable); 4305d91b1b49SAndrew Gallatin TUNABLE_INT_FETCH("hw.mxge.force_firmware", 4306d91b1b49SAndrew Gallatin &mxge_force_firmware); 43075e7d8541SAndrew Gallatin TUNABLE_INT_FETCH("hw.mxge.deassert_wait", 43085e7d8541SAndrew Gallatin &mxge_deassert_wait); 43095e7d8541SAndrew Gallatin TUNABLE_INT_FETCH("hw.mxge.verbose", 43105e7d8541SAndrew Gallatin &mxge_verbose); 4311dce01b9bSAndrew Gallatin TUNABLE_INT_FETCH("hw.mxge.ticks", &mxge_ticks); 43121e413cf9SAndrew Gallatin TUNABLE_INT_FETCH("hw.mxge.always_promisc", &mxge_always_promisc); 43131e413cf9SAndrew Gallatin TUNABLE_INT_FETCH("hw.mxge.rss_hash_type", &mxge_rss_hash_type); 431494c7d993SAndrew Gallatin TUNABLE_INT_FETCH("hw.mxge.rss_hashtype", &mxge_rss_hash_type); 4315f9453025SAndrew Gallatin TUNABLE_INT_FETCH("hw.mxge.initial_mtu", &mxge_initial_mtu); 431665c69066SAndrew Gallatin TUNABLE_INT_FETCH("hw.mxge.throttle", &mxge_throttle); 4317b2fc195eSAndrew Gallatin 43185e7d8541SAndrew Gallatin if (bootverbose) 43195e7d8541SAndrew Gallatin mxge_verbose = 1; 43206d87a65dSAndrew Gallatin if (mxge_intr_coal_delay < 0 || mxge_intr_coal_delay > 10*1000) 43216d87a65dSAndrew Gallatin mxge_intr_coal_delay = 30; 4322dce01b9bSAndrew Gallatin if (mxge_ticks == 0) 43231e413cf9SAndrew Gallatin mxge_ticks = hz / 2; 43246d87a65dSAndrew Gallatin sc->pause = mxge_flow_control; 43251e413cf9SAndrew Gallatin if (mxge_rss_hash_type < MXGEFW_RSS_HASH_TYPE_IPV4 4326bb8ddc66SAndrew Gallatin || mxge_rss_hash_type > MXGEFW_RSS_HASH_TYPE_MAX) { 43275769c5efSAndrew Gallatin mxge_rss_hash_type = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT; 4328b2fc195eSAndrew Gallatin } 4329f9453025SAndrew Gallatin if (mxge_initial_mtu > ETHERMTU_JUMBO || 4330f9453025SAndrew Gallatin mxge_initial_mtu < ETHER_MIN_LEN) 4331f9453025SAndrew Gallatin mxge_initial_mtu = ETHERMTU_JUMBO; 433265c69066SAndrew Gallatin 433365c69066SAndrew Gallatin if (mxge_throttle && mxge_throttle > MXGE_MAX_THROTTLE) 433465c69066SAndrew Gallatin mxge_throttle = MXGE_MAX_THROTTLE; 433565c69066SAndrew Gallatin if (mxge_throttle && mxge_throttle < MXGE_MIN_THROTTLE) 433665c69066SAndrew Gallatin mxge_throttle = MXGE_MIN_THROTTLE; 433765c69066SAndrew Gallatin sc->throttle = mxge_throttle; 43381e413cf9SAndrew Gallatin } 43391e413cf9SAndrew Gallatin 43401e413cf9SAndrew Gallatin 43411e413cf9SAndrew Gallatin static void 43421e413cf9SAndrew Gallatin mxge_free_slices(mxge_softc_t *sc) 43431e413cf9SAndrew Gallatin { 43441e413cf9SAndrew Gallatin struct mxge_slice_state *ss; 43451e413cf9SAndrew Gallatin int i; 43461e413cf9SAndrew Gallatin 43471e413cf9SAndrew Gallatin 43481e413cf9SAndrew Gallatin if (sc->ss == NULL) 43491e413cf9SAndrew Gallatin return; 43501e413cf9SAndrew Gallatin 43511e413cf9SAndrew Gallatin for (i = 0; i < sc->num_slices; i++) { 43521e413cf9SAndrew Gallatin ss = &sc->ss[i]; 43531e413cf9SAndrew Gallatin if (ss->fw_stats != NULL) { 43541e413cf9SAndrew Gallatin mxge_dma_free(&ss->fw_stats_dma); 43551e413cf9SAndrew Gallatin ss->fw_stats = NULL; 4356c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING 4357c6cb3e3fSAndrew Gallatin if (ss->tx.br != NULL) { 4358c6cb3e3fSAndrew Gallatin drbr_free(ss->tx.br, M_DEVBUF); 4359c6cb3e3fSAndrew Gallatin ss->tx.br = NULL; 4360c6cb3e3fSAndrew Gallatin } 4361c6cb3e3fSAndrew Gallatin #endif 43621e413cf9SAndrew Gallatin mtx_destroy(&ss->tx.mtx); 43631e413cf9SAndrew Gallatin } 43641e413cf9SAndrew Gallatin if (ss->rx_done.entry != NULL) { 43651e413cf9SAndrew Gallatin mxge_dma_free(&ss->rx_done.dma); 43661e413cf9SAndrew Gallatin ss->rx_done.entry = NULL; 43671e413cf9SAndrew Gallatin } 43681e413cf9SAndrew Gallatin } 43691e413cf9SAndrew Gallatin free(sc->ss, M_DEVBUF); 43701e413cf9SAndrew Gallatin sc->ss = NULL; 43711e413cf9SAndrew Gallatin } 43721e413cf9SAndrew Gallatin 43731e413cf9SAndrew Gallatin static int 43741e413cf9SAndrew Gallatin mxge_alloc_slices(mxge_softc_t *sc) 43751e413cf9SAndrew Gallatin { 43761e413cf9SAndrew Gallatin mxge_cmd_t cmd; 43771e413cf9SAndrew Gallatin struct mxge_slice_state *ss; 43781e413cf9SAndrew Gallatin size_t bytes; 43791e413cf9SAndrew Gallatin int err, i, max_intr_slots; 43801e413cf9SAndrew Gallatin 43811e413cf9SAndrew Gallatin err = mxge_send_cmd(sc, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd); 43821e413cf9SAndrew Gallatin if (err != 0) { 43831e413cf9SAndrew Gallatin device_printf(sc->dev, "Cannot determine rx ring size\n"); 43841e413cf9SAndrew Gallatin return err; 43851e413cf9SAndrew Gallatin } 43861e413cf9SAndrew Gallatin sc->rx_ring_size = cmd.data0; 43871e413cf9SAndrew Gallatin max_intr_slots = 2 * (sc->rx_ring_size / sizeof (mcp_dma_addr_t)); 43881e413cf9SAndrew Gallatin 4389ac2fffa4SPedro F. Giffuni bytes = sizeof (*sc->ss) * sc->num_slices; 4390ac2fffa4SPedro F. Giffuni sc->ss = malloc(bytes, M_DEVBUF, M_NOWAIT | M_ZERO); 43911e413cf9SAndrew Gallatin if (sc->ss == NULL) 43921e413cf9SAndrew Gallatin return (ENOMEM); 43931e413cf9SAndrew Gallatin for (i = 0; i < sc->num_slices; i++) { 43941e413cf9SAndrew Gallatin ss = &sc->ss[i]; 43951e413cf9SAndrew Gallatin 43961e413cf9SAndrew Gallatin ss->sc = sc; 43971e413cf9SAndrew Gallatin 43981e413cf9SAndrew Gallatin /* allocate per-slice rx interrupt queues */ 43991e413cf9SAndrew Gallatin 44001e413cf9SAndrew Gallatin bytes = max_intr_slots * sizeof (*ss->rx_done.entry); 44011e413cf9SAndrew Gallatin err = mxge_dma_alloc(sc, &ss->rx_done.dma, bytes, 4096); 44021e413cf9SAndrew Gallatin if (err != 0) 44031e413cf9SAndrew Gallatin goto abort; 44041e413cf9SAndrew Gallatin ss->rx_done.entry = ss->rx_done.dma.addr; 44051e413cf9SAndrew Gallatin bzero(ss->rx_done.entry, bytes); 44061e413cf9SAndrew Gallatin 44071e413cf9SAndrew Gallatin /* 44081e413cf9SAndrew Gallatin * allocate the per-slice firmware stats; stats 44091e413cf9SAndrew Gallatin * (including tx) are used used only on the first 44101e413cf9SAndrew Gallatin * slice for now 44111e413cf9SAndrew Gallatin */ 4412c6cb3e3fSAndrew Gallatin #ifndef IFNET_BUF_RING 44131e413cf9SAndrew Gallatin if (i > 0) 44141e413cf9SAndrew Gallatin continue; 4415c6cb3e3fSAndrew Gallatin #endif 44161e413cf9SAndrew Gallatin 44171e413cf9SAndrew Gallatin bytes = sizeof (*ss->fw_stats); 44181e413cf9SAndrew Gallatin err = mxge_dma_alloc(sc, &ss->fw_stats_dma, 44191e413cf9SAndrew Gallatin sizeof (*ss->fw_stats), 64); 44201e413cf9SAndrew Gallatin if (err != 0) 44211e413cf9SAndrew Gallatin goto abort; 44221e413cf9SAndrew Gallatin ss->fw_stats = (mcp_irq_data_t *)ss->fw_stats_dma.addr; 44231e413cf9SAndrew Gallatin snprintf(ss->tx.mtx_name, sizeof(ss->tx.mtx_name), 44241e413cf9SAndrew Gallatin "%s:tx(%d)", device_get_nameunit(sc->dev), i); 44251e413cf9SAndrew Gallatin mtx_init(&ss->tx.mtx, ss->tx.mtx_name, NULL, MTX_DEF); 4426c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING 4427c6cb3e3fSAndrew Gallatin ss->tx.br = buf_ring_alloc(2048, M_DEVBUF, M_WAITOK, 4428c6cb3e3fSAndrew Gallatin &ss->tx.mtx); 4429c6cb3e3fSAndrew Gallatin #endif 44301e413cf9SAndrew Gallatin } 44311e413cf9SAndrew Gallatin 44321e413cf9SAndrew Gallatin return (0); 44331e413cf9SAndrew Gallatin 44341e413cf9SAndrew Gallatin abort: 44351e413cf9SAndrew Gallatin mxge_free_slices(sc); 44361e413cf9SAndrew Gallatin return (ENOMEM); 44371e413cf9SAndrew Gallatin } 44381e413cf9SAndrew Gallatin 44391e413cf9SAndrew Gallatin static void 44401e413cf9SAndrew Gallatin mxge_slice_probe(mxge_softc_t *sc) 44411e413cf9SAndrew Gallatin { 44421e413cf9SAndrew Gallatin mxge_cmd_t cmd; 44431e413cf9SAndrew Gallatin char *old_fw; 44441e413cf9SAndrew Gallatin int msix_cnt, status, max_intr_slots; 44451e413cf9SAndrew Gallatin 44461e413cf9SAndrew Gallatin sc->num_slices = 1; 44471e413cf9SAndrew Gallatin /* 44481e413cf9SAndrew Gallatin * don't enable multiple slices if they are not enabled, 44491e413cf9SAndrew Gallatin * or if this is not an SMP system 44501e413cf9SAndrew Gallatin */ 44511e413cf9SAndrew Gallatin 44521e413cf9SAndrew Gallatin if (mxge_max_slices == 0 || mxge_max_slices == 1 || mp_ncpus < 2) 44531e413cf9SAndrew Gallatin return; 44541e413cf9SAndrew Gallatin 44551e413cf9SAndrew Gallatin /* see how many MSI-X interrupts are available */ 44561e413cf9SAndrew Gallatin msix_cnt = pci_msix_count(sc->dev); 44571e413cf9SAndrew Gallatin if (msix_cnt < 2) 44581e413cf9SAndrew Gallatin return; 44591e413cf9SAndrew Gallatin 44601e413cf9SAndrew Gallatin /* now load the slice aware firmware see what it supports */ 44611e413cf9SAndrew Gallatin old_fw = sc->fw_name; 44621e413cf9SAndrew Gallatin if (old_fw == mxge_fw_aligned) 44631e413cf9SAndrew Gallatin sc->fw_name = mxge_fw_rss_aligned; 44641e413cf9SAndrew Gallatin else 44651e413cf9SAndrew Gallatin sc->fw_name = mxge_fw_rss_unaligned; 44661e413cf9SAndrew Gallatin status = mxge_load_firmware(sc, 0); 44671e413cf9SAndrew Gallatin if (status != 0) { 44681e413cf9SAndrew Gallatin device_printf(sc->dev, "Falling back to a single slice\n"); 44691e413cf9SAndrew Gallatin return; 44701e413cf9SAndrew Gallatin } 44711e413cf9SAndrew Gallatin 44721e413cf9SAndrew Gallatin /* try to send a reset command to the card to see if it 44731e413cf9SAndrew Gallatin is alive */ 44741e413cf9SAndrew Gallatin memset(&cmd, 0, sizeof (cmd)); 44751e413cf9SAndrew Gallatin status = mxge_send_cmd(sc, MXGEFW_CMD_RESET, &cmd); 44761e413cf9SAndrew Gallatin if (status != 0) { 44771e413cf9SAndrew Gallatin device_printf(sc->dev, "failed reset\n"); 44781e413cf9SAndrew Gallatin goto abort_with_fw; 44791e413cf9SAndrew Gallatin } 44801e413cf9SAndrew Gallatin 44811e413cf9SAndrew Gallatin /* get rx ring size */ 44821e413cf9SAndrew Gallatin status = mxge_send_cmd(sc, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd); 44831e413cf9SAndrew Gallatin if (status != 0) { 44841e413cf9SAndrew Gallatin device_printf(sc->dev, "Cannot determine rx ring size\n"); 44851e413cf9SAndrew Gallatin goto abort_with_fw; 44861e413cf9SAndrew Gallatin } 44871e413cf9SAndrew Gallatin max_intr_slots = 2 * (cmd.data0 / sizeof (mcp_dma_addr_t)); 44881e413cf9SAndrew Gallatin 44891e413cf9SAndrew Gallatin /* tell it the size of the interrupt queues */ 44901e413cf9SAndrew Gallatin cmd.data0 = max_intr_slots * sizeof (struct mcp_slot); 44911e413cf9SAndrew Gallatin status = mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd); 44921e413cf9SAndrew Gallatin if (status != 0) { 44931e413cf9SAndrew Gallatin device_printf(sc->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n"); 44941e413cf9SAndrew Gallatin goto abort_with_fw; 44951e413cf9SAndrew Gallatin } 44961e413cf9SAndrew Gallatin 44971e413cf9SAndrew Gallatin /* ask the maximum number of slices it supports */ 44981e413cf9SAndrew Gallatin status = mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd); 44991e413cf9SAndrew Gallatin if (status != 0) { 45001e413cf9SAndrew Gallatin device_printf(sc->dev, 45011e413cf9SAndrew Gallatin "failed MXGEFW_CMD_GET_MAX_RSS_QUEUES\n"); 45021e413cf9SAndrew Gallatin goto abort_with_fw; 45031e413cf9SAndrew Gallatin } 45041e413cf9SAndrew Gallatin sc->num_slices = cmd.data0; 45051e413cf9SAndrew Gallatin if (sc->num_slices > msix_cnt) 45061e413cf9SAndrew Gallatin sc->num_slices = msix_cnt; 45071e413cf9SAndrew Gallatin 45081e413cf9SAndrew Gallatin if (mxge_max_slices == -1) { 45091e413cf9SAndrew Gallatin /* cap to number of CPUs in system */ 45101e413cf9SAndrew Gallatin if (sc->num_slices > mp_ncpus) 45111e413cf9SAndrew Gallatin sc->num_slices = mp_ncpus; 45121e413cf9SAndrew Gallatin } else { 45131e413cf9SAndrew Gallatin if (sc->num_slices > mxge_max_slices) 45141e413cf9SAndrew Gallatin sc->num_slices = mxge_max_slices; 45151e413cf9SAndrew Gallatin } 45161e413cf9SAndrew Gallatin /* make sure it is a power of two */ 45171e413cf9SAndrew Gallatin while (sc->num_slices & (sc->num_slices - 1)) 45181e413cf9SAndrew Gallatin sc->num_slices--; 45191e413cf9SAndrew Gallatin 45201e413cf9SAndrew Gallatin if (mxge_verbose) 45211e413cf9SAndrew Gallatin device_printf(sc->dev, "using %d slices\n", 45221e413cf9SAndrew Gallatin sc->num_slices); 45231e413cf9SAndrew Gallatin 45241e413cf9SAndrew Gallatin return; 45251e413cf9SAndrew Gallatin 45261e413cf9SAndrew Gallatin abort_with_fw: 45271e413cf9SAndrew Gallatin sc->fw_name = old_fw; 45281e413cf9SAndrew Gallatin (void) mxge_load_firmware(sc, 0); 45291e413cf9SAndrew Gallatin } 45301e413cf9SAndrew Gallatin 45311e413cf9SAndrew Gallatin static int 45321e413cf9SAndrew Gallatin mxge_add_msix_irqs(mxge_softc_t *sc) 45331e413cf9SAndrew Gallatin { 4534ac2fffa4SPedro F. Giffuni size_t bytes; 45351e413cf9SAndrew Gallatin int count, err, i, rid; 45361e413cf9SAndrew Gallatin 45371e413cf9SAndrew Gallatin rid = PCIR_BAR(2); 45381e413cf9SAndrew Gallatin sc->msix_table_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 45391e413cf9SAndrew Gallatin &rid, RF_ACTIVE); 45401e413cf9SAndrew Gallatin 45411e413cf9SAndrew Gallatin if (sc->msix_table_res == NULL) { 45421e413cf9SAndrew Gallatin device_printf(sc->dev, "couldn't alloc MSIX table res\n"); 45431e413cf9SAndrew Gallatin return ENXIO; 45441e413cf9SAndrew Gallatin } 45451e413cf9SAndrew Gallatin 45461e413cf9SAndrew Gallatin count = sc->num_slices; 45471e413cf9SAndrew Gallatin err = pci_alloc_msix(sc->dev, &count); 45481e413cf9SAndrew Gallatin if (err != 0) { 45491e413cf9SAndrew Gallatin device_printf(sc->dev, "pci_alloc_msix: failed, wanted %d" 45501e413cf9SAndrew Gallatin "err = %d \n", sc->num_slices, err); 45511e413cf9SAndrew Gallatin goto abort_with_msix_table; 45521e413cf9SAndrew Gallatin } 45531e413cf9SAndrew Gallatin if (count < sc->num_slices) { 45541e413cf9SAndrew Gallatin device_printf(sc->dev, "pci_alloc_msix: need %d, got %d\n", 45551e413cf9SAndrew Gallatin count, sc->num_slices); 45561e413cf9SAndrew Gallatin device_printf(sc->dev, 45571e413cf9SAndrew Gallatin "Try setting hw.mxge.max_slices to %d\n", 45581e413cf9SAndrew Gallatin count); 45591e413cf9SAndrew Gallatin err = ENOSPC; 45601e413cf9SAndrew Gallatin goto abort_with_msix; 45611e413cf9SAndrew Gallatin } 4562ac2fffa4SPedro F. Giffuni bytes = sizeof (*sc->msix_irq_res) * sc->num_slices; 4563ac2fffa4SPedro F. Giffuni sc->msix_irq_res = malloc(bytes, M_DEVBUF, M_NOWAIT|M_ZERO); 45641e413cf9SAndrew Gallatin if (sc->msix_irq_res == NULL) { 45651e413cf9SAndrew Gallatin err = ENOMEM; 45661e413cf9SAndrew Gallatin goto abort_with_msix; 45671e413cf9SAndrew Gallatin } 45681e413cf9SAndrew Gallatin 45691e413cf9SAndrew Gallatin for (i = 0; i < sc->num_slices; i++) { 45701e413cf9SAndrew Gallatin rid = i + 1; 45711e413cf9SAndrew Gallatin sc->msix_irq_res[i] = bus_alloc_resource_any(sc->dev, 45721e413cf9SAndrew Gallatin SYS_RES_IRQ, 45731e413cf9SAndrew Gallatin &rid, RF_ACTIVE); 45741e413cf9SAndrew Gallatin if (sc->msix_irq_res[i] == NULL) { 45751e413cf9SAndrew Gallatin device_printf(sc->dev, "couldn't allocate IRQ res" 45761e413cf9SAndrew Gallatin " for message %d\n", i); 45771e413cf9SAndrew Gallatin err = ENXIO; 45781e413cf9SAndrew Gallatin goto abort_with_res; 45791e413cf9SAndrew Gallatin } 45801e413cf9SAndrew Gallatin } 45811e413cf9SAndrew Gallatin 4582ac2fffa4SPedro F. Giffuni bytes = sizeof (*sc->msix_ih) * sc->num_slices; 4583ac2fffa4SPedro F. Giffuni sc->msix_ih = malloc(bytes, M_DEVBUF, M_NOWAIT|M_ZERO); 45841e413cf9SAndrew Gallatin 45851e413cf9SAndrew Gallatin for (i = 0; i < sc->num_slices; i++) { 45861e413cf9SAndrew Gallatin err = bus_setup_intr(sc->dev, sc->msix_irq_res[i], 45871e413cf9SAndrew Gallatin INTR_TYPE_NET | INTR_MPSAFE, 458837d89b0cSAndrew Gallatin #if __FreeBSD_version > 700030 458937d89b0cSAndrew Gallatin NULL, 459037d89b0cSAndrew Gallatin #endif 459137d89b0cSAndrew Gallatin mxge_intr, &sc->ss[i], &sc->msix_ih[i]); 45921e413cf9SAndrew Gallatin if (err != 0) { 45931e413cf9SAndrew Gallatin device_printf(sc->dev, "couldn't setup intr for " 45941e413cf9SAndrew Gallatin "message %d\n", i); 45951e413cf9SAndrew Gallatin goto abort_with_intr; 45961e413cf9SAndrew Gallatin } 459721089137SAndrew Gallatin bus_describe_intr(sc->dev, sc->msix_irq_res[i], 459821089137SAndrew Gallatin sc->msix_ih[i], "s%d", i); 45991e413cf9SAndrew Gallatin } 46001e413cf9SAndrew Gallatin 46011e413cf9SAndrew Gallatin if (mxge_verbose) { 46021e413cf9SAndrew Gallatin device_printf(sc->dev, "using %d msix IRQs:", 46031e413cf9SAndrew Gallatin sc->num_slices); 46041e413cf9SAndrew Gallatin for (i = 0; i < sc->num_slices; i++) 4605da1b038aSJustin Hibbits printf(" %jd", rman_get_start(sc->msix_irq_res[i])); 46061e413cf9SAndrew Gallatin printf("\n"); 46071e413cf9SAndrew Gallatin } 46081e413cf9SAndrew Gallatin return (0); 46091e413cf9SAndrew Gallatin 46101e413cf9SAndrew Gallatin abort_with_intr: 46111e413cf9SAndrew Gallatin for (i = 0; i < sc->num_slices; i++) { 46121e413cf9SAndrew Gallatin if (sc->msix_ih[i] != NULL) { 46131e413cf9SAndrew Gallatin bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 46141e413cf9SAndrew Gallatin sc->msix_ih[i]); 46151e413cf9SAndrew Gallatin sc->msix_ih[i] = NULL; 46161e413cf9SAndrew Gallatin } 46171e413cf9SAndrew Gallatin } 46181e413cf9SAndrew Gallatin free(sc->msix_ih, M_DEVBUF); 46191e413cf9SAndrew Gallatin 46201e413cf9SAndrew Gallatin 46211e413cf9SAndrew Gallatin abort_with_res: 46221e413cf9SAndrew Gallatin for (i = 0; i < sc->num_slices; i++) { 46231e413cf9SAndrew Gallatin rid = i + 1; 46241e413cf9SAndrew Gallatin if (sc->msix_irq_res[i] != NULL) 46251e413cf9SAndrew Gallatin bus_release_resource(sc->dev, SYS_RES_IRQ, rid, 46261e413cf9SAndrew Gallatin sc->msix_irq_res[i]); 46271e413cf9SAndrew Gallatin sc->msix_irq_res[i] = NULL; 46281e413cf9SAndrew Gallatin } 46291e413cf9SAndrew Gallatin free(sc->msix_irq_res, M_DEVBUF); 46301e413cf9SAndrew Gallatin 46311e413cf9SAndrew Gallatin 46321e413cf9SAndrew Gallatin abort_with_msix: 46331e413cf9SAndrew Gallatin pci_release_msi(sc->dev); 46341e413cf9SAndrew Gallatin 46351e413cf9SAndrew Gallatin abort_with_msix_table: 46361e413cf9SAndrew Gallatin bus_release_resource(sc->dev, SYS_RES_MEMORY, PCIR_BAR(2), 46371e413cf9SAndrew Gallatin sc->msix_table_res); 46381e413cf9SAndrew Gallatin 46391e413cf9SAndrew Gallatin return err; 46401e413cf9SAndrew Gallatin } 46411e413cf9SAndrew Gallatin 46421e413cf9SAndrew Gallatin static int 46431e413cf9SAndrew Gallatin mxge_add_single_irq(mxge_softc_t *sc) 46441e413cf9SAndrew Gallatin { 46451e413cf9SAndrew Gallatin int count, err, rid; 46461e413cf9SAndrew Gallatin 46471e413cf9SAndrew Gallatin count = pci_msi_count(sc->dev); 46481e413cf9SAndrew Gallatin if (count == 1 && pci_alloc_msi(sc->dev, &count) == 0) { 46491e413cf9SAndrew Gallatin rid = 1; 46501e413cf9SAndrew Gallatin } else { 46511e413cf9SAndrew Gallatin rid = 0; 465291ed8913SAndrew Gallatin sc->legacy_irq = 1; 46531e413cf9SAndrew Gallatin } 465443cd6160SJustin Hibbits sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid, 465543cd6160SJustin Hibbits RF_SHAREABLE | RF_ACTIVE); 46561e413cf9SAndrew Gallatin if (sc->irq_res == NULL) { 46571e413cf9SAndrew Gallatin device_printf(sc->dev, "could not alloc interrupt\n"); 46581e413cf9SAndrew Gallatin return ENXIO; 46591e413cf9SAndrew Gallatin } 46601e413cf9SAndrew Gallatin if (mxge_verbose) 4661da1b038aSJustin Hibbits device_printf(sc->dev, "using %s irq %jd\n", 466291ed8913SAndrew Gallatin sc->legacy_irq ? "INTx" : "MSI", 46631e413cf9SAndrew Gallatin rman_get_start(sc->irq_res)); 46641e413cf9SAndrew Gallatin err = bus_setup_intr(sc->dev, sc->irq_res, 46651e413cf9SAndrew Gallatin INTR_TYPE_NET | INTR_MPSAFE, 466637d89b0cSAndrew Gallatin #if __FreeBSD_version > 700030 466737d89b0cSAndrew Gallatin NULL, 466837d89b0cSAndrew Gallatin #endif 466937d89b0cSAndrew Gallatin mxge_intr, &sc->ss[0], &sc->ih); 46701e413cf9SAndrew Gallatin if (err != 0) { 46711e413cf9SAndrew Gallatin bus_release_resource(sc->dev, SYS_RES_IRQ, 467291ed8913SAndrew Gallatin sc->legacy_irq ? 0 : 1, sc->irq_res); 467391ed8913SAndrew Gallatin if (!sc->legacy_irq) 46741e413cf9SAndrew Gallatin pci_release_msi(sc->dev); 46751e413cf9SAndrew Gallatin } 46761e413cf9SAndrew Gallatin return err; 46771e413cf9SAndrew Gallatin } 46781e413cf9SAndrew Gallatin 46791e413cf9SAndrew Gallatin static void 46801e413cf9SAndrew Gallatin mxge_rem_msix_irqs(mxge_softc_t *sc) 46811e413cf9SAndrew Gallatin { 46821e413cf9SAndrew Gallatin int i, rid; 46831e413cf9SAndrew Gallatin 46841e413cf9SAndrew Gallatin for (i = 0; i < sc->num_slices; i++) { 46851e413cf9SAndrew Gallatin if (sc->msix_ih[i] != NULL) { 46861e413cf9SAndrew Gallatin bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 46871e413cf9SAndrew Gallatin sc->msix_ih[i]); 46881e413cf9SAndrew Gallatin sc->msix_ih[i] = NULL; 46891e413cf9SAndrew Gallatin } 46901e413cf9SAndrew Gallatin } 46911e413cf9SAndrew Gallatin free(sc->msix_ih, M_DEVBUF); 46921e413cf9SAndrew Gallatin 46931e413cf9SAndrew Gallatin for (i = 0; i < sc->num_slices; i++) { 46941e413cf9SAndrew Gallatin rid = i + 1; 46951e413cf9SAndrew Gallatin if (sc->msix_irq_res[i] != NULL) 46961e413cf9SAndrew Gallatin bus_release_resource(sc->dev, SYS_RES_IRQ, rid, 46971e413cf9SAndrew Gallatin sc->msix_irq_res[i]); 46981e413cf9SAndrew Gallatin sc->msix_irq_res[i] = NULL; 46991e413cf9SAndrew Gallatin } 47001e413cf9SAndrew Gallatin free(sc->msix_irq_res, M_DEVBUF); 47011e413cf9SAndrew Gallatin 47021e413cf9SAndrew Gallatin bus_release_resource(sc->dev, SYS_RES_MEMORY, PCIR_BAR(2), 47031e413cf9SAndrew Gallatin sc->msix_table_res); 47041e413cf9SAndrew Gallatin 47051e413cf9SAndrew Gallatin pci_release_msi(sc->dev); 47061e413cf9SAndrew Gallatin return; 47071e413cf9SAndrew Gallatin } 47081e413cf9SAndrew Gallatin 47091e413cf9SAndrew Gallatin static void 47101e413cf9SAndrew Gallatin mxge_rem_single_irq(mxge_softc_t *sc) 47111e413cf9SAndrew Gallatin { 47121e413cf9SAndrew Gallatin bus_teardown_intr(sc->dev, sc->irq_res, sc->ih); 47131e413cf9SAndrew Gallatin bus_release_resource(sc->dev, SYS_RES_IRQ, 471491ed8913SAndrew Gallatin sc->legacy_irq ? 0 : 1, sc->irq_res); 471591ed8913SAndrew Gallatin if (!sc->legacy_irq) 47161e413cf9SAndrew Gallatin pci_release_msi(sc->dev); 47171e413cf9SAndrew Gallatin } 47181e413cf9SAndrew Gallatin 47191e413cf9SAndrew Gallatin static void 47201e413cf9SAndrew Gallatin mxge_rem_irq(mxge_softc_t *sc) 47211e413cf9SAndrew Gallatin { 47221e413cf9SAndrew Gallatin if (sc->num_slices > 1) 47231e413cf9SAndrew Gallatin mxge_rem_msix_irqs(sc); 47241e413cf9SAndrew Gallatin else 47251e413cf9SAndrew Gallatin mxge_rem_single_irq(sc); 47261e413cf9SAndrew Gallatin } 47271e413cf9SAndrew Gallatin 47281e413cf9SAndrew Gallatin static int 47291e413cf9SAndrew Gallatin mxge_add_irq(mxge_softc_t *sc) 47301e413cf9SAndrew Gallatin { 47311e413cf9SAndrew Gallatin int err; 47321e413cf9SAndrew Gallatin 47331e413cf9SAndrew Gallatin if (sc->num_slices > 1) 47341e413cf9SAndrew Gallatin err = mxge_add_msix_irqs(sc); 47351e413cf9SAndrew Gallatin else 47361e413cf9SAndrew Gallatin err = mxge_add_single_irq(sc); 47371e413cf9SAndrew Gallatin 47381e413cf9SAndrew Gallatin if (0 && err == 0 && sc->num_slices > 1) { 47391e413cf9SAndrew Gallatin mxge_rem_msix_irqs(sc); 47401e413cf9SAndrew Gallatin err = mxge_add_msix_irqs(sc); 47411e413cf9SAndrew Gallatin } 47421e413cf9SAndrew Gallatin return err; 47431e413cf9SAndrew Gallatin } 47441e413cf9SAndrew Gallatin 4745b2fc195eSAndrew Gallatin 4746b2fc195eSAndrew Gallatin static int 47476d87a65dSAndrew Gallatin mxge_attach(device_t dev) 4748b2fc195eSAndrew Gallatin { 47490a7a780eSAndrew Gallatin mxge_cmd_t cmd; 47506d87a65dSAndrew Gallatin mxge_softc_t *sc = device_get_softc(dev); 4751b2fc195eSAndrew Gallatin struct ifnet *ifp; 47521e413cf9SAndrew Gallatin int err, rid; 4753b2fc195eSAndrew Gallatin 4754b2fc195eSAndrew Gallatin sc->dev = dev; 47556d87a65dSAndrew Gallatin mxge_fetch_tunables(sc); 4756b2fc195eSAndrew Gallatin 475772c042dfSAndrew Gallatin TASK_INIT(&sc->watchdog_task, 1, mxge_watchdog_task, sc); 4758a4a75d67SJohn Baldwin sc->tq = taskqueue_create("mxge_taskq", M_WAITOK, 4759a4a75d67SJohn Baldwin taskqueue_thread_enqueue, &sc->tq); 476072c042dfSAndrew Gallatin if (sc->tq == NULL) { 476172c042dfSAndrew Gallatin err = ENOMEM; 476272c042dfSAndrew Gallatin goto abort_with_nothing; 476372c042dfSAndrew Gallatin } 476472c042dfSAndrew Gallatin 476562ce43ccSScott Long err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 4766b2fc195eSAndrew Gallatin 1, /* alignment */ 47671e413cf9SAndrew Gallatin 0, /* boundary */ 4768b2fc195eSAndrew Gallatin BUS_SPACE_MAXADDR, /* low */ 4769b2fc195eSAndrew Gallatin BUS_SPACE_MAXADDR, /* high */ 4770b2fc195eSAndrew Gallatin NULL, NULL, /* filter */ 4771aed8e389SAndrew Gallatin 65536 + 256, /* maxsize */ 47725e7d8541SAndrew Gallatin MXGE_MAX_SEND_DESC, /* num segs */ 47731e413cf9SAndrew Gallatin 65536, /* maxsegsize */ 4774b2fc195eSAndrew Gallatin 0, /* flags */ 4775b2fc195eSAndrew Gallatin NULL, NULL, /* lock */ 4776b2fc195eSAndrew Gallatin &sc->parent_dmat); /* tag */ 4777b2fc195eSAndrew Gallatin 4778b2fc195eSAndrew Gallatin if (err != 0) { 4779b2fc195eSAndrew Gallatin device_printf(sc->dev, "Err %d allocating parent dmat\n", 4780b2fc195eSAndrew Gallatin err); 478172c042dfSAndrew Gallatin goto abort_with_tq; 4782b2fc195eSAndrew Gallatin } 4783b2fc195eSAndrew Gallatin 4784b2fc195eSAndrew Gallatin ifp = sc->ifp = if_alloc(IFT_ETHER); 4785b2fc195eSAndrew Gallatin if (ifp == NULL) { 4786b2fc195eSAndrew Gallatin device_printf(dev, "can not if_alloc()\n"); 4787b2fc195eSAndrew Gallatin err = ENOSPC; 4788b2fc195eSAndrew Gallatin goto abort_with_parent_dmat; 4789b2fc195eSAndrew Gallatin } 47901e413cf9SAndrew Gallatin if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 47911e413cf9SAndrew Gallatin 4792a98d6cd7SAndrew Gallatin snprintf(sc->cmd_mtx_name, sizeof(sc->cmd_mtx_name), "%s:cmd", 4793a98d6cd7SAndrew Gallatin device_get_nameunit(dev)); 4794a98d6cd7SAndrew Gallatin mtx_init(&sc->cmd_mtx, sc->cmd_mtx_name, NULL, MTX_DEF); 4795a98d6cd7SAndrew Gallatin snprintf(sc->driver_mtx_name, sizeof(sc->driver_mtx_name), 4796a98d6cd7SAndrew Gallatin "%s:drv", device_get_nameunit(dev)); 4797a98d6cd7SAndrew Gallatin mtx_init(&sc->driver_mtx, sc->driver_mtx_name, 4798b2fc195eSAndrew Gallatin MTX_NETWORK_LOCK, MTX_DEF); 4799b2fc195eSAndrew Gallatin 4800dce01b9bSAndrew Gallatin callout_init_mtx(&sc->co_hdl, &sc->driver_mtx, 0); 4801d91b1b49SAndrew Gallatin 4802dce01b9bSAndrew Gallatin mxge_setup_cfg_space(sc); 4803b2fc195eSAndrew Gallatin 4804b2fc195eSAndrew Gallatin /* Map the board into the kernel */ 4805b2fc195eSAndrew Gallatin rid = PCIR_BARS; 480643cd6160SJustin Hibbits sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 480743cd6160SJustin Hibbits RF_ACTIVE); 4808b2fc195eSAndrew Gallatin if (sc->mem_res == NULL) { 4809b2fc195eSAndrew Gallatin device_printf(dev, "could not map memory\n"); 4810b2fc195eSAndrew Gallatin err = ENXIO; 4811b2fc195eSAndrew Gallatin goto abort_with_lock; 4812b2fc195eSAndrew Gallatin } 4813b2fc195eSAndrew Gallatin sc->sram = rman_get_virtual(sc->mem_res); 4814b2fc195eSAndrew Gallatin sc->sram_size = 2*1024*1024 - (2*(48*1024)+(32*1024)) - 0x100; 4815b2fc195eSAndrew Gallatin if (sc->sram_size > rman_get_size(sc->mem_res)) { 4816da1b038aSJustin Hibbits device_printf(dev, "impossible memory region size %jd\n", 4817b2fc195eSAndrew Gallatin rman_get_size(sc->mem_res)); 4818b2fc195eSAndrew Gallatin err = ENXIO; 4819b2fc195eSAndrew Gallatin goto abort_with_mem_res; 4820b2fc195eSAndrew Gallatin } 4821b2fc195eSAndrew Gallatin 4822b2fc195eSAndrew Gallatin /* make NULL terminated copy of the EEPROM strings section of 4823b2fc195eSAndrew Gallatin lanai SRAM */ 48246d87a65dSAndrew Gallatin bzero(sc->eeprom_strings, MXGE_EEPROM_STRINGS_SIZE); 4825b2fc195eSAndrew Gallatin bus_space_read_region_1(rman_get_bustag(sc->mem_res), 4826b2fc195eSAndrew Gallatin rman_get_bushandle(sc->mem_res), 48276d87a65dSAndrew Gallatin sc->sram_size - MXGE_EEPROM_STRINGS_SIZE, 4828b2fc195eSAndrew Gallatin sc->eeprom_strings, 48296d87a65dSAndrew Gallatin MXGE_EEPROM_STRINGS_SIZE - 2); 48306d87a65dSAndrew Gallatin err = mxge_parse_strings(sc); 4831b2fc195eSAndrew Gallatin if (err != 0) 4832b2fc195eSAndrew Gallatin goto abort_with_mem_res; 4833b2fc195eSAndrew Gallatin 4834b2fc195eSAndrew Gallatin /* Enable write combining for efficient use of PCIe bus */ 48356d87a65dSAndrew Gallatin mxge_enable_wc(sc); 4836b2fc195eSAndrew Gallatin 4837b2fc195eSAndrew Gallatin /* Allocate the out of band dma memory */ 48386d87a65dSAndrew Gallatin err = mxge_dma_alloc(sc, &sc->cmd_dma, 48396d87a65dSAndrew Gallatin sizeof (mxge_cmd_t), 64); 4840b2fc195eSAndrew Gallatin if (err != 0) 4841b2fc195eSAndrew Gallatin goto abort_with_mem_res; 4842b2fc195eSAndrew Gallatin sc->cmd = (mcp_cmd_response_t *) sc->cmd_dma.addr; 48436d87a65dSAndrew Gallatin err = mxge_dma_alloc(sc, &sc->zeropad_dma, 64, 64); 4844b2fc195eSAndrew Gallatin if (err != 0) 4845b2fc195eSAndrew Gallatin goto abort_with_cmd_dma; 4846b2fc195eSAndrew Gallatin 4847a98d6cd7SAndrew Gallatin err = mxge_dma_alloc(sc, &sc->dmabench_dma, 4096, 4096); 4848a98d6cd7SAndrew Gallatin if (err != 0) 48491e413cf9SAndrew Gallatin goto abort_with_zeropad_dma; 4850b2fc195eSAndrew Gallatin 48518fe615baSAndrew Gallatin /* select & load the firmware */ 48528fe615baSAndrew Gallatin err = mxge_select_firmware(sc); 4853b2fc195eSAndrew Gallatin if (err != 0) 48541e413cf9SAndrew Gallatin goto abort_with_dmabench; 48555e7d8541SAndrew Gallatin sc->intr_coal_delay = mxge_intr_coal_delay; 48561e413cf9SAndrew Gallatin 48571e413cf9SAndrew Gallatin mxge_slice_probe(sc); 48581e413cf9SAndrew Gallatin err = mxge_alloc_slices(sc); 48591e413cf9SAndrew Gallatin if (err != 0) 48601e413cf9SAndrew Gallatin goto abort_with_dmabench; 48611e413cf9SAndrew Gallatin 4862adae7080SAndrew Gallatin err = mxge_reset(sc, 0); 4863b2fc195eSAndrew Gallatin if (err != 0) 48641e413cf9SAndrew Gallatin goto abort_with_slices; 4865b2fc195eSAndrew Gallatin 4866a98d6cd7SAndrew Gallatin err = mxge_alloc_rings(sc); 4867a98d6cd7SAndrew Gallatin if (err != 0) { 4868a98d6cd7SAndrew Gallatin device_printf(sc->dev, "failed to allocate rings\n"); 48692e084798SAndrew Gallatin goto abort_with_slices; 4870a98d6cd7SAndrew Gallatin } 4871a98d6cd7SAndrew Gallatin 48721e413cf9SAndrew Gallatin err = mxge_add_irq(sc); 4873a98d6cd7SAndrew Gallatin if (err != 0) { 48741e413cf9SAndrew Gallatin device_printf(sc->dev, "failed to add irq\n"); 4875a98d6cd7SAndrew Gallatin goto abort_with_rings; 4876a98d6cd7SAndrew Gallatin } 48771e413cf9SAndrew Gallatin 4878b245f96cSGleb Smirnoff ifp->if_baudrate = IF_Gbps(10); 4879c792928fSAndrew Gallatin ifp->if_capabilities = IFCAP_RXCSUM | IFCAP_TXCSUM | IFCAP_TSO4 | 488026dd49c6SAndrew Gallatin IFCAP_VLAN_MTU | IFCAP_LINKSTATE | IFCAP_TXCSUM_IPV6 | 488126dd49c6SAndrew Gallatin IFCAP_RXCSUM_IPV6; 488226dd49c6SAndrew Gallatin #if defined(INET) || defined(INET6) 4883eb6219e3SAndrew Gallatin ifp->if_capabilities |= IFCAP_LRO; 4884eb6219e3SAndrew Gallatin #endif 488537d89b0cSAndrew Gallatin 488637d89b0cSAndrew Gallatin #ifdef MXGE_NEW_VLAN_API 488737d89b0cSAndrew Gallatin ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM; 48880dce6781SAndrew Gallatin 48890dce6781SAndrew Gallatin /* Only FW 1.4.32 and newer can do TSO over vlans */ 48900dce6781SAndrew Gallatin if (sc->fw_ver_major == 1 && sc->fw_ver_minor == 4 && 48910dce6781SAndrew Gallatin sc->fw_ver_tiny >= 32) 48920dce6781SAndrew Gallatin ifp->if_capabilities |= IFCAP_VLAN_HWTSO; 489337d89b0cSAndrew Gallatin #endif 4894053e637fSAndrew Gallatin sc->max_mtu = mxge_max_mtu(sc); 4895053e637fSAndrew Gallatin if (sc->max_mtu >= 9000) 4896053e637fSAndrew Gallatin ifp->if_capabilities |= IFCAP_JUMBO_MTU; 4897053e637fSAndrew Gallatin else 4898053e637fSAndrew Gallatin device_printf(dev, "MTU limited to %d. Install " 4899adae7080SAndrew Gallatin "latest firmware for 9000 byte jumbo support\n", 4900053e637fSAndrew Gallatin sc->max_mtu - ETHER_HDR_LEN); 4901aed8e389SAndrew Gallatin ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_TSO; 49020a7a780eSAndrew Gallatin ifp->if_hwassist |= CSUM_TCP_IPV6 | CSUM_UDP_IPV6; 49030a7a780eSAndrew Gallatin /* check to see if f/w supports TSO for IPv6 */ 49040a7a780eSAndrew Gallatin if (!mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_TSO6_HDR_SIZE, &cmd)) { 49050a7a780eSAndrew Gallatin if (CSUM_TCP_IPV6) 49060a7a780eSAndrew Gallatin ifp->if_capabilities |= IFCAP_TSO6; 49070a7a780eSAndrew Gallatin sc->max_tso6_hlen = min(cmd.data0, 49080a7a780eSAndrew Gallatin sizeof (sc->ss[0].scratch)); 49090a7a780eSAndrew Gallatin } 4910b2fc195eSAndrew Gallatin ifp->if_capenable = ifp->if_capabilities; 4911f04b33f8SAndrew Gallatin if (sc->lro_cnt == 0) 4912f04b33f8SAndrew Gallatin ifp->if_capenable &= ~IFCAP_LRO; 49136d87a65dSAndrew Gallatin ifp->if_init = mxge_init; 4914b2fc195eSAndrew Gallatin ifp->if_softc = sc; 4915b2fc195eSAndrew Gallatin ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 49166d87a65dSAndrew Gallatin ifp->if_ioctl = mxge_ioctl; 49176d87a65dSAndrew Gallatin ifp->if_start = mxge_start; 4918f3f040d9SGleb Smirnoff ifp->if_get_counter = mxge_get_counter; 4919c587e59fSAndrew Gallatin /* Initialise the ifmedia structure */ 4920c587e59fSAndrew Gallatin ifmedia_init(&sc->media, 0, mxge_media_change, 4921c587e59fSAndrew Gallatin mxge_media_status); 4922c406ad2eSAndrew Gallatin mxge_media_init(sc); 4923c587e59fSAndrew Gallatin mxge_media_probe(sc); 49248c5d766cSAndrew Gallatin sc->dying = 0; 4925b2fc195eSAndrew Gallatin ether_ifattach(ifp, sc->mac_addr); 4926f9453025SAndrew Gallatin /* ether_ifattach sets mtu to ETHERMTU */ 4927f9453025SAndrew Gallatin if (mxge_initial_mtu != ETHERMTU) 4928f9453025SAndrew Gallatin mxge_change_mtu(sc, mxge_initial_mtu); 4929b2fc195eSAndrew Gallatin 49306d87a65dSAndrew Gallatin mxge_add_sysctls(sc); 4931c6cb3e3fSAndrew Gallatin #ifdef IFNET_BUF_RING 4932c6cb3e3fSAndrew Gallatin ifp->if_transmit = mxge_transmit; 4933c6cb3e3fSAndrew Gallatin ifp->if_qflush = mxge_qflush; 4934c6cb3e3fSAndrew Gallatin #endif 49352e084798SAndrew Gallatin taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq", 49362e084798SAndrew Gallatin device_get_nameunit(sc->dev)); 49376b484a49SAndrew Gallatin callout_reset(&sc->co_hdl, mxge_ticks, mxge_tick, sc); 4938b2fc195eSAndrew Gallatin return 0; 4939b2fc195eSAndrew Gallatin 4940a98d6cd7SAndrew Gallatin abort_with_rings: 4941a98d6cd7SAndrew Gallatin mxge_free_rings(sc); 49421e413cf9SAndrew Gallatin abort_with_slices: 49431e413cf9SAndrew Gallatin mxge_free_slices(sc); 4944a98d6cd7SAndrew Gallatin abort_with_dmabench: 4945a98d6cd7SAndrew Gallatin mxge_dma_free(&sc->dmabench_dma); 4946b2fc195eSAndrew Gallatin abort_with_zeropad_dma: 49476d87a65dSAndrew Gallatin mxge_dma_free(&sc->zeropad_dma); 4948b2fc195eSAndrew Gallatin abort_with_cmd_dma: 49496d87a65dSAndrew Gallatin mxge_dma_free(&sc->cmd_dma); 4950b2fc195eSAndrew Gallatin abort_with_mem_res: 4951b2fc195eSAndrew Gallatin bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BARS, sc->mem_res); 4952b2fc195eSAndrew Gallatin abort_with_lock: 4953b2fc195eSAndrew Gallatin pci_disable_busmaster(dev); 4954a98d6cd7SAndrew Gallatin mtx_destroy(&sc->cmd_mtx); 4955a98d6cd7SAndrew Gallatin mtx_destroy(&sc->driver_mtx); 4956b2fc195eSAndrew Gallatin if_free(ifp); 4957b2fc195eSAndrew Gallatin abort_with_parent_dmat: 4958b2fc195eSAndrew Gallatin bus_dma_tag_destroy(sc->parent_dmat); 495972c042dfSAndrew Gallatin abort_with_tq: 496072c042dfSAndrew Gallatin if (sc->tq != NULL) { 496172c042dfSAndrew Gallatin taskqueue_drain(sc->tq, &sc->watchdog_task); 496272c042dfSAndrew Gallatin taskqueue_free(sc->tq); 496372c042dfSAndrew Gallatin sc->tq = NULL; 496472c042dfSAndrew Gallatin } 4965b2fc195eSAndrew Gallatin abort_with_nothing: 4966b2fc195eSAndrew Gallatin return err; 4967b2fc195eSAndrew Gallatin } 4968b2fc195eSAndrew Gallatin 4969b2fc195eSAndrew Gallatin static int 49706d87a65dSAndrew Gallatin mxge_detach(device_t dev) 4971b2fc195eSAndrew Gallatin { 49726d87a65dSAndrew Gallatin mxge_softc_t *sc = device_get_softc(dev); 4973b2fc195eSAndrew Gallatin 497437d89b0cSAndrew Gallatin if (mxge_vlans_active(sc)) { 4975c792928fSAndrew Gallatin device_printf(sc->dev, 4976c792928fSAndrew Gallatin "Detach vlans before removing module\n"); 4977c792928fSAndrew Gallatin return EBUSY; 4978c792928fSAndrew Gallatin } 4979a98d6cd7SAndrew Gallatin mtx_lock(&sc->driver_mtx); 49808c5d766cSAndrew Gallatin sc->dying = 1; 4981b2fc195eSAndrew Gallatin if (sc->ifp->if_drv_flags & IFF_DRV_RUNNING) 4982a393336bSAndrew Gallatin mxge_close(sc, 0); 4983a98d6cd7SAndrew Gallatin mtx_unlock(&sc->driver_mtx); 4984b2fc195eSAndrew Gallatin ether_ifdetach(sc->ifp); 498572c042dfSAndrew Gallatin if (sc->tq != NULL) { 498672c042dfSAndrew Gallatin taskqueue_drain(sc->tq, &sc->watchdog_task); 498772c042dfSAndrew Gallatin taskqueue_free(sc->tq); 498872c042dfSAndrew Gallatin sc->tq = NULL; 498972c042dfSAndrew Gallatin } 4990e749ef6bSAndrew Gallatin callout_drain(&sc->co_hdl); 4991dce01b9bSAndrew Gallatin ifmedia_removeall(&sc->media); 4992091feecdSAndrew Gallatin mxge_dummy_rdma(sc, 0); 49931e413cf9SAndrew Gallatin mxge_rem_sysctls(sc); 49941e413cf9SAndrew Gallatin mxge_rem_irq(sc); 4995a98d6cd7SAndrew Gallatin mxge_free_rings(sc); 49961e413cf9SAndrew Gallatin mxge_free_slices(sc); 4997a98d6cd7SAndrew Gallatin mxge_dma_free(&sc->dmabench_dma); 49986d87a65dSAndrew Gallatin mxge_dma_free(&sc->zeropad_dma); 49996d87a65dSAndrew Gallatin mxge_dma_free(&sc->cmd_dma); 5000b2fc195eSAndrew Gallatin bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BARS, sc->mem_res); 5001b2fc195eSAndrew Gallatin pci_disable_busmaster(dev); 5002a98d6cd7SAndrew Gallatin mtx_destroy(&sc->cmd_mtx); 5003a98d6cd7SAndrew Gallatin mtx_destroy(&sc->driver_mtx); 5004b2fc195eSAndrew Gallatin if_free(sc->ifp); 5005b2fc195eSAndrew Gallatin bus_dma_tag_destroy(sc->parent_dmat); 5006b2fc195eSAndrew Gallatin return 0; 5007b2fc195eSAndrew Gallatin } 5008b2fc195eSAndrew Gallatin 5009b2fc195eSAndrew Gallatin static int 50106d87a65dSAndrew Gallatin mxge_shutdown(device_t dev) 5011b2fc195eSAndrew Gallatin { 5012b2fc195eSAndrew Gallatin return 0; 5013b2fc195eSAndrew Gallatin } 5014b2fc195eSAndrew Gallatin 5015b2fc195eSAndrew Gallatin /* 5016b2fc195eSAndrew Gallatin This file uses Myri10GE driver indentation. 5017b2fc195eSAndrew Gallatin 5018b2fc195eSAndrew Gallatin Local Variables: 5019b2fc195eSAndrew Gallatin c-file-style:"linux" 5020b2fc195eSAndrew Gallatin tab-width:8 5021b2fc195eSAndrew Gallatin End: 5022b2fc195eSAndrew Gallatin */ 5023