1a9e644cdSLuigi Rizzo /*- 2a9e644cdSLuigi Rizzo * Copyright (c) 2016, Vincenzo Maffione 3a9e644cdSLuigi Rizzo * All rights reserved. 4a9e644cdSLuigi Rizzo * 5a9e644cdSLuigi Rizzo * Redistribution and use in source and binary forms, with or without 6a9e644cdSLuigi Rizzo * modification, are permitted provided that the following conditions 7a9e644cdSLuigi Rizzo * are met: 8a9e644cdSLuigi Rizzo * 1. Redistributions of source code must retain the above copyright 9a9e644cdSLuigi Rizzo * notice unmodified, this list of conditions, and the following 10a9e644cdSLuigi Rizzo * disclaimer. 11a9e644cdSLuigi Rizzo * 2. Redistributions in binary form must reproduce the above copyright 12a9e644cdSLuigi Rizzo * notice, this list of conditions and the following disclaimer in the 13a9e644cdSLuigi Rizzo * documentation and/or other materials provided with the distribution. 14a9e644cdSLuigi Rizzo * 15a9e644cdSLuigi Rizzo * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16a9e644cdSLuigi Rizzo * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17a9e644cdSLuigi Rizzo * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18a9e644cdSLuigi Rizzo * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19a9e644cdSLuigi Rizzo * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20a9e644cdSLuigi Rizzo * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21a9e644cdSLuigi Rizzo * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22a9e644cdSLuigi Rizzo * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23a9e644cdSLuigi Rizzo * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24a9e644cdSLuigi Rizzo * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25a9e644cdSLuigi Rizzo * 26a9e644cdSLuigi Rizzo * $FreeBSD$ 27a9e644cdSLuigi Rizzo */ 28a9e644cdSLuigi Rizzo 29a9e644cdSLuigi Rizzo /* Driver for ptnet paravirtualized network device. */ 30a9e644cdSLuigi Rizzo 31a9e644cdSLuigi Rizzo #include <sys/cdefs.h> 32a9e644cdSLuigi Rizzo 33a9e644cdSLuigi Rizzo #include <sys/types.h> 34a9e644cdSLuigi Rizzo #include <sys/param.h> 35a9e644cdSLuigi Rizzo #include <sys/systm.h> 36a9e644cdSLuigi Rizzo #include <sys/kernel.h> 37a9e644cdSLuigi Rizzo #include <sys/sockio.h> 38a9e644cdSLuigi Rizzo #include <sys/mbuf.h> 39a9e644cdSLuigi Rizzo #include <sys/malloc.h> 40a9e644cdSLuigi Rizzo #include <sys/module.h> 41a9e644cdSLuigi Rizzo #include <sys/socket.h> 42a9e644cdSLuigi Rizzo #include <sys/sysctl.h> 43a9e644cdSLuigi Rizzo #include <sys/lock.h> 44a9e644cdSLuigi Rizzo #include <sys/mutex.h> 45a9e644cdSLuigi Rizzo #include <sys/taskqueue.h> 46a9e644cdSLuigi Rizzo #include <sys/smp.h> 47a9e644cdSLuigi Rizzo #include <sys/time.h> 48a9e644cdSLuigi Rizzo #include <machine/smp.h> 49a9e644cdSLuigi Rizzo 50a9e644cdSLuigi Rizzo #include <vm/uma.h> 51a9e644cdSLuigi Rizzo #include <vm/vm.h> 52a9e644cdSLuigi Rizzo #include <vm/pmap.h> 53a9e644cdSLuigi Rizzo 54a9e644cdSLuigi Rizzo #include <net/ethernet.h> 55a9e644cdSLuigi Rizzo #include <net/if.h> 56a9e644cdSLuigi Rizzo #include <net/if_var.h> 57a9e644cdSLuigi Rizzo #include <net/if_arp.h> 58a9e644cdSLuigi Rizzo #include <net/if_dl.h> 59a9e644cdSLuigi Rizzo #include <net/if_types.h> 60a9e644cdSLuigi Rizzo #include <net/if_media.h> 61a9e644cdSLuigi Rizzo #include <net/if_vlan_var.h> 62a9e644cdSLuigi Rizzo #include <net/bpf.h> 63a9e644cdSLuigi Rizzo 64a9e644cdSLuigi Rizzo #include <netinet/in_systm.h> 65a9e644cdSLuigi Rizzo #include <netinet/in.h> 66a9e644cdSLuigi Rizzo #include <netinet/ip.h> 67a9e644cdSLuigi Rizzo #include <netinet/ip6.h> 68a9e644cdSLuigi Rizzo #include <netinet6/ip6_var.h> 69a9e644cdSLuigi Rizzo #include <netinet/udp.h> 70a9e644cdSLuigi Rizzo #include <netinet/tcp.h> 71a9e644cdSLuigi Rizzo 72a9e644cdSLuigi Rizzo #include <machine/bus.h> 73a9e644cdSLuigi Rizzo #include <machine/resource.h> 74a9e644cdSLuigi Rizzo #include <sys/bus.h> 75a9e644cdSLuigi Rizzo #include <sys/rman.h> 76a9e644cdSLuigi Rizzo 77a9e644cdSLuigi Rizzo #include <dev/pci/pcivar.h> 78a9e644cdSLuigi Rizzo #include <dev/pci/pcireg.h> 79a9e644cdSLuigi Rizzo 80a9e644cdSLuigi Rizzo #include "opt_inet.h" 81a9e644cdSLuigi Rizzo #include "opt_inet6.h" 82a9e644cdSLuigi Rizzo 83a9e644cdSLuigi Rizzo #include <sys/selinfo.h> 84a9e644cdSLuigi Rizzo #include <net/netmap.h> 85a9e644cdSLuigi Rizzo #include <dev/netmap/netmap_kern.h> 86a9e644cdSLuigi Rizzo #include <net/netmap_virt.h> 87a9e644cdSLuigi Rizzo #include <dev/netmap/netmap_mem2.h> 88a9e644cdSLuigi Rizzo #include <dev/virtio/network/virtio_net.h> 89a9e644cdSLuigi Rizzo 90a9e644cdSLuigi Rizzo #ifndef INET 91a9e644cdSLuigi Rizzo #error "INET not defined, cannot support offloadings" 92a9e644cdSLuigi Rizzo #endif 93a9e644cdSLuigi Rizzo 94a9e644cdSLuigi Rizzo #if __FreeBSD_version >= 1100000 95a9e644cdSLuigi Rizzo static uint64_t ptnet_get_counter(if_t, ift_counter); 96a9e644cdSLuigi Rizzo #else 97a9e644cdSLuigi Rizzo typedef struct ifnet *if_t; 98a9e644cdSLuigi Rizzo #define if_getsoftc(_ifp) (_ifp)->if_softc 99a9e644cdSLuigi Rizzo #endif 100a9e644cdSLuigi Rizzo 101a9e644cdSLuigi Rizzo //#define PTNETMAP_STATS 102a9e644cdSLuigi Rizzo //#define DEBUG 103a9e644cdSLuigi Rizzo #ifdef DEBUG 104a9e644cdSLuigi Rizzo #define DBG(x) x 105a9e644cdSLuigi Rizzo #else /* !DEBUG */ 106a9e644cdSLuigi Rizzo #define DBG(x) 107a9e644cdSLuigi Rizzo #endif /* !DEBUG */ 108a9e644cdSLuigi Rizzo 109a9e644cdSLuigi Rizzo extern int ptnet_vnet_hdr; /* Tunable parameter */ 110a9e644cdSLuigi Rizzo 111a9e644cdSLuigi Rizzo struct ptnet_softc; 112a9e644cdSLuigi Rizzo 113a9e644cdSLuigi Rizzo struct ptnet_queue_stats { 114a9e644cdSLuigi Rizzo uint64_t packets; /* if_[io]packets */ 115a9e644cdSLuigi Rizzo uint64_t bytes; /* if_[io]bytes */ 116a9e644cdSLuigi Rizzo uint64_t errors; /* if_[io]errors */ 117a9e644cdSLuigi Rizzo uint64_t iqdrops; /* if_iqdrops */ 118a9e644cdSLuigi Rizzo uint64_t mcasts; /* if_[io]mcasts */ 119a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS 120a9e644cdSLuigi Rizzo uint64_t intrs; 121a9e644cdSLuigi Rizzo uint64_t kicks; 122a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */ 123a9e644cdSLuigi Rizzo }; 124a9e644cdSLuigi Rizzo 125a9e644cdSLuigi Rizzo struct ptnet_queue { 126a9e644cdSLuigi Rizzo struct ptnet_softc *sc; 127a9e644cdSLuigi Rizzo struct resource *irq; 128a9e644cdSLuigi Rizzo void *cookie; 129a9e644cdSLuigi Rizzo int kring_id; 130b6e66be2SVincenzo Maffione struct nm_csb_atok *atok; 131b6e66be2SVincenzo Maffione struct nm_csb_ktoa *ktoa; 132a9e644cdSLuigi Rizzo unsigned int kick; 133a9e644cdSLuigi Rizzo struct mtx lock; 134a9e644cdSLuigi Rizzo struct buf_ring *bufring; /* for TX queues */ 135a9e644cdSLuigi Rizzo struct ptnet_queue_stats stats; 136a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS 137a9e644cdSLuigi Rizzo struct ptnet_queue_stats last_stats; 138a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */ 139a9e644cdSLuigi Rizzo struct taskqueue *taskq; 140a9e644cdSLuigi Rizzo struct task task; 141a9e644cdSLuigi Rizzo char lock_name[16]; 142a9e644cdSLuigi Rizzo }; 143a9e644cdSLuigi Rizzo 144a9e644cdSLuigi Rizzo #define PTNET_Q_LOCK(_pq) mtx_lock(&(_pq)->lock) 145a9e644cdSLuigi Rizzo #define PTNET_Q_TRYLOCK(_pq) mtx_trylock(&(_pq)->lock) 146a9e644cdSLuigi Rizzo #define PTNET_Q_UNLOCK(_pq) mtx_unlock(&(_pq)->lock) 147a9e644cdSLuigi Rizzo 148a9e644cdSLuigi Rizzo struct ptnet_softc { 149a9e644cdSLuigi Rizzo device_t dev; 150a9e644cdSLuigi Rizzo if_t ifp; 151a9e644cdSLuigi Rizzo struct ifmedia media; 152a9e644cdSLuigi Rizzo struct mtx lock; 153a9e644cdSLuigi Rizzo char lock_name[16]; 154a9e644cdSLuigi Rizzo char hwaddr[ETHER_ADDR_LEN]; 155a9e644cdSLuigi Rizzo 156a9e644cdSLuigi Rizzo /* Mirror of PTFEAT register. */ 157a9e644cdSLuigi Rizzo uint32_t ptfeatures; 158a9e644cdSLuigi Rizzo unsigned int vnet_hdr_len; 159a9e644cdSLuigi Rizzo 160a9e644cdSLuigi Rizzo /* PCI BARs support. */ 161a9e644cdSLuigi Rizzo struct resource *iomem; 162a9e644cdSLuigi Rizzo struct resource *msix_mem; 163a9e644cdSLuigi Rizzo 164a9e644cdSLuigi Rizzo unsigned int num_rings; 165a9e644cdSLuigi Rizzo unsigned int num_tx_rings; 166a9e644cdSLuigi Rizzo struct ptnet_queue *queues; 167a9e644cdSLuigi Rizzo struct ptnet_queue *rxqueues; 168b6e66be2SVincenzo Maffione struct nm_csb_atok *csb_gh; 169b6e66be2SVincenzo Maffione struct nm_csb_ktoa *csb_hg; 170a9e644cdSLuigi Rizzo 171a9e644cdSLuigi Rizzo unsigned int min_tx_space; 172a9e644cdSLuigi Rizzo 173a9e644cdSLuigi Rizzo struct netmap_pt_guest_adapter *ptna; 174a9e644cdSLuigi Rizzo 175a9e644cdSLuigi Rizzo struct callout tick; 176a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS 177a9e644cdSLuigi Rizzo struct timeval last_ts; 178a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */ 179a9e644cdSLuigi Rizzo }; 180a9e644cdSLuigi Rizzo 181a9e644cdSLuigi Rizzo #define PTNET_CORE_LOCK(_sc) mtx_lock(&(_sc)->lock) 182a9e644cdSLuigi Rizzo #define PTNET_CORE_UNLOCK(_sc) mtx_unlock(&(_sc)->lock) 183a9e644cdSLuigi Rizzo 184a9e644cdSLuigi Rizzo static int ptnet_probe(device_t); 185a9e644cdSLuigi Rizzo static int ptnet_attach(device_t); 186a9e644cdSLuigi Rizzo static int ptnet_detach(device_t); 187a9e644cdSLuigi Rizzo static int ptnet_suspend(device_t); 188a9e644cdSLuigi Rizzo static int ptnet_resume(device_t); 189a9e644cdSLuigi Rizzo static int ptnet_shutdown(device_t); 190a9e644cdSLuigi Rizzo 191a9e644cdSLuigi Rizzo static void ptnet_init(void *opaque); 192a9e644cdSLuigi Rizzo static int ptnet_ioctl(if_t ifp, u_long cmd, caddr_t data); 193a9e644cdSLuigi Rizzo static int ptnet_init_locked(struct ptnet_softc *sc); 194a9e644cdSLuigi Rizzo static int ptnet_stop(struct ptnet_softc *sc); 195a9e644cdSLuigi Rizzo static int ptnet_transmit(if_t ifp, struct mbuf *m); 196a9e644cdSLuigi Rizzo static int ptnet_drain_transmit_queue(struct ptnet_queue *pq, 197a9e644cdSLuigi Rizzo unsigned int budget, 198a9e644cdSLuigi Rizzo bool may_resched); 199a9e644cdSLuigi Rizzo static void ptnet_qflush(if_t ifp); 200a9e644cdSLuigi Rizzo static void ptnet_tx_task(void *context, int pending); 201a9e644cdSLuigi Rizzo 202a9e644cdSLuigi Rizzo static int ptnet_media_change(if_t ifp); 203a9e644cdSLuigi Rizzo static void ptnet_media_status(if_t ifp, struct ifmediareq *ifmr); 204a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS 205a9e644cdSLuigi Rizzo static void ptnet_tick(void *opaque); 206a9e644cdSLuigi Rizzo #endif 207a9e644cdSLuigi Rizzo 208a9e644cdSLuigi Rizzo static int ptnet_irqs_init(struct ptnet_softc *sc); 209a9e644cdSLuigi Rizzo static void ptnet_irqs_fini(struct ptnet_softc *sc); 210a9e644cdSLuigi Rizzo 211b6e66be2SVincenzo Maffione static uint32_t ptnet_nm_ptctl(struct ptnet_softc *sc, uint32_t cmd); 2122ff91c17SVincenzo Maffione static int ptnet_nm_config(struct netmap_adapter *na, 2132ff91c17SVincenzo Maffione struct nm_config_info *info); 214a9e644cdSLuigi Rizzo static void ptnet_update_vnet_hdr(struct ptnet_softc *sc); 215a9e644cdSLuigi Rizzo static int ptnet_nm_register(struct netmap_adapter *na, int onoff); 216a9e644cdSLuigi Rizzo static int ptnet_nm_txsync(struct netmap_kring *kring, int flags); 217a9e644cdSLuigi Rizzo static int ptnet_nm_rxsync(struct netmap_kring *kring, int flags); 2184f80b14cSVincenzo Maffione static void ptnet_nm_intr(struct netmap_adapter *na, int onoff); 219a9e644cdSLuigi Rizzo 220a9e644cdSLuigi Rizzo static void ptnet_tx_intr(void *opaque); 221a9e644cdSLuigi Rizzo static void ptnet_rx_intr(void *opaque); 222a9e644cdSLuigi Rizzo 223a9e644cdSLuigi Rizzo static unsigned ptnet_rx_discard(struct netmap_kring *kring, 224a9e644cdSLuigi Rizzo unsigned int head); 225a9e644cdSLuigi Rizzo static int ptnet_rx_eof(struct ptnet_queue *pq, unsigned int budget, 226a9e644cdSLuigi Rizzo bool may_resched); 227a9e644cdSLuigi Rizzo static void ptnet_rx_task(void *context, int pending); 228a9e644cdSLuigi Rizzo 229a9e644cdSLuigi Rizzo #ifdef DEVICE_POLLING 230a9e644cdSLuigi Rizzo static poll_handler_t ptnet_poll; 231a9e644cdSLuigi Rizzo #endif 232a9e644cdSLuigi Rizzo 233a9e644cdSLuigi Rizzo static device_method_t ptnet_methods[] = { 234a9e644cdSLuigi Rizzo DEVMETHOD(device_probe, ptnet_probe), 235a9e644cdSLuigi Rizzo DEVMETHOD(device_attach, ptnet_attach), 236a9e644cdSLuigi Rizzo DEVMETHOD(device_detach, ptnet_detach), 237a9e644cdSLuigi Rizzo DEVMETHOD(device_suspend, ptnet_suspend), 238a9e644cdSLuigi Rizzo DEVMETHOD(device_resume, ptnet_resume), 239a9e644cdSLuigi Rizzo DEVMETHOD(device_shutdown, ptnet_shutdown), 240a9e644cdSLuigi Rizzo DEVMETHOD_END 241a9e644cdSLuigi Rizzo }; 242a9e644cdSLuigi Rizzo 243a9e644cdSLuigi Rizzo static driver_t ptnet_driver = { 244a9e644cdSLuigi Rizzo "ptnet", 245a9e644cdSLuigi Rizzo ptnet_methods, 246a9e644cdSLuigi Rizzo sizeof(struct ptnet_softc) 247a9e644cdSLuigi Rizzo }; 248a9e644cdSLuigi Rizzo 249a9e644cdSLuigi Rizzo /* We use (SI_ORDER_MIDDLE+2) here, see DEV_MODULE_ORDERED() invocation. */ 250a9e644cdSLuigi Rizzo static devclass_t ptnet_devclass; 251a9e644cdSLuigi Rizzo DRIVER_MODULE_ORDERED(ptnet, pci, ptnet_driver, ptnet_devclass, 252a9e644cdSLuigi Rizzo NULL, NULL, SI_ORDER_MIDDLE + 2); 253a9e644cdSLuigi Rizzo 254a9e644cdSLuigi Rizzo static int 255a9e644cdSLuigi Rizzo ptnet_probe(device_t dev) 256a9e644cdSLuigi Rizzo { 257a9e644cdSLuigi Rizzo if (pci_get_vendor(dev) != PTNETMAP_PCI_VENDOR_ID || 258a9e644cdSLuigi Rizzo pci_get_device(dev) != PTNETMAP_PCI_NETIF_ID) { 259a9e644cdSLuigi Rizzo return (ENXIO); 260a9e644cdSLuigi Rizzo } 261a9e644cdSLuigi Rizzo 262a9e644cdSLuigi Rizzo device_set_desc(dev, "ptnet network adapter"); 263a9e644cdSLuigi Rizzo 264a9e644cdSLuigi Rizzo return (BUS_PROBE_DEFAULT); 265a9e644cdSLuigi Rizzo } 266a9e644cdSLuigi Rizzo 267a9e644cdSLuigi Rizzo static inline void ptnet_kick(struct ptnet_queue *pq) 268a9e644cdSLuigi Rizzo { 269a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS 270a9e644cdSLuigi Rizzo pq->stats.kicks ++; 271a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */ 272a9e644cdSLuigi Rizzo bus_write_4(pq->sc->iomem, pq->kick, 0); 273a9e644cdSLuigi Rizzo } 274a9e644cdSLuigi Rizzo 275a9e644cdSLuigi Rizzo #define PTNET_BUF_RING_SIZE 4096 276a9e644cdSLuigi Rizzo #define PTNET_RX_BUDGET 512 277a9e644cdSLuigi Rizzo #define PTNET_RX_BATCH 1 278a9e644cdSLuigi Rizzo #define PTNET_TX_BUDGET 512 279a9e644cdSLuigi Rizzo #define PTNET_TX_BATCH 64 280a9e644cdSLuigi Rizzo #define PTNET_HDR_SIZE sizeof(struct virtio_net_hdr_mrg_rxbuf) 281a9e644cdSLuigi Rizzo #define PTNET_MAX_PKT_SIZE 65536 282a9e644cdSLuigi Rizzo 283d337c8c7SVincenzo Maffione #define PTNET_CSUM_OFFLOAD (CSUM_TCP | CSUM_UDP) 284d337c8c7SVincenzo Maffione #define PTNET_CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6 | CSUM_UDP_IPV6) 285a9e644cdSLuigi Rizzo #define PTNET_ALL_OFFLOAD (CSUM_TSO | PTNET_CSUM_OFFLOAD |\ 286a9e644cdSLuigi Rizzo PTNET_CSUM_OFFLOAD_IPV6) 287a9e644cdSLuigi Rizzo 288a9e644cdSLuigi Rizzo static int 289a9e644cdSLuigi Rizzo ptnet_attach(device_t dev) 290a9e644cdSLuigi Rizzo { 291844a6f0cSLuigi Rizzo uint32_t ptfeatures = 0; 292a9e644cdSLuigi Rizzo unsigned int num_rx_rings, num_tx_rings; 293a9e644cdSLuigi Rizzo struct netmap_adapter na_arg; 294a9e644cdSLuigi Rizzo unsigned int nifp_offset; 295a9e644cdSLuigi Rizzo struct ptnet_softc *sc; 296a9e644cdSLuigi Rizzo if_t ifp; 297a9e644cdSLuigi Rizzo uint32_t macreg; 298a9e644cdSLuigi Rizzo int err, rid; 299a9e644cdSLuigi Rizzo int i; 300a9e644cdSLuigi Rizzo 301a9e644cdSLuigi Rizzo sc = device_get_softc(dev); 302a9e644cdSLuigi Rizzo sc->dev = dev; 303a9e644cdSLuigi Rizzo 304a9e644cdSLuigi Rizzo /* Setup PCI resources. */ 305a9e644cdSLuigi Rizzo pci_enable_busmaster(dev); 306a9e644cdSLuigi Rizzo 307a9e644cdSLuigi Rizzo rid = PCIR_BAR(PTNETMAP_IO_PCI_BAR); 308a9e644cdSLuigi Rizzo sc->iomem = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid, 309a9e644cdSLuigi Rizzo RF_ACTIVE); 310a9e644cdSLuigi Rizzo if (sc->iomem == NULL) { 311a9e644cdSLuigi Rizzo device_printf(dev, "Failed to map I/O BAR\n"); 312a9e644cdSLuigi Rizzo return (ENXIO); 313a9e644cdSLuigi Rizzo } 314a9e644cdSLuigi Rizzo 315844a6f0cSLuigi Rizzo /* Negotiate features with the hypervisor. */ 316a9e644cdSLuigi Rizzo if (ptnet_vnet_hdr) { 317a9e644cdSLuigi Rizzo ptfeatures |= PTNETMAP_F_VNET_HDR; 318a9e644cdSLuigi Rizzo } 319a9e644cdSLuigi Rizzo bus_write_4(sc->iomem, PTNET_IO_PTFEAT, ptfeatures); /* wanted */ 320a9e644cdSLuigi Rizzo ptfeatures = bus_read_4(sc->iomem, PTNET_IO_PTFEAT); /* acked */ 321a9e644cdSLuigi Rizzo sc->ptfeatures = ptfeatures; 322a9e644cdSLuigi Rizzo 32346023447SVincenzo Maffione num_tx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_RINGS); 32446023447SVincenzo Maffione num_rx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_RINGS); 32546023447SVincenzo Maffione sc->num_rings = num_tx_rings + num_rx_rings; 32646023447SVincenzo Maffione sc->num_tx_rings = num_tx_rings; 32746023447SVincenzo Maffione 328b6e66be2SVincenzo Maffione if (sc->num_rings * sizeof(struct nm_csb_atok) > PAGE_SIZE) { 32946023447SVincenzo Maffione device_printf(dev, "CSB cannot handle that many rings (%u)\n", 33046023447SVincenzo Maffione sc->num_rings); 33146023447SVincenzo Maffione err = ENOMEM; 33246023447SVincenzo Maffione goto err_path; 33346023447SVincenzo Maffione } 33446023447SVincenzo Maffione 33546023447SVincenzo Maffione /* Allocate CSB and carry out CSB allocation protocol. */ 33646023447SVincenzo Maffione sc->csb_gh = contigmalloc(2*PAGE_SIZE, M_DEVBUF, M_NOWAIT | M_ZERO, 33746023447SVincenzo Maffione (size_t)0, -1UL, PAGE_SIZE, 0); 33846023447SVincenzo Maffione if (sc->csb_gh == NULL) { 339a9e644cdSLuigi Rizzo device_printf(dev, "Failed to allocate CSB\n"); 340a9e644cdSLuigi Rizzo err = ENOMEM; 341a9e644cdSLuigi Rizzo goto err_path; 342a9e644cdSLuigi Rizzo } 343b6e66be2SVincenzo Maffione sc->csb_hg = (struct nm_csb_ktoa *)(((char *)sc->csb_gh) + PAGE_SIZE); 344a9e644cdSLuigi Rizzo 345a9e644cdSLuigi Rizzo { 346a2a74091SLuigi Rizzo /* 347a2a74091SLuigi Rizzo * We use uint64_t rather than vm_paddr_t since we 348a2a74091SLuigi Rizzo * need 64 bit addresses even on 32 bit platforms. 349a2a74091SLuigi Rizzo */ 35046023447SVincenzo Maffione uint64_t paddr = vtophys(sc->csb_gh); 351a9e644cdSLuigi Rizzo 35246023447SVincenzo Maffione /* CSB allocation protocol: write to BAH first, then 35346023447SVincenzo Maffione * to BAL (for both GH and HG sections). */ 35446023447SVincenzo Maffione bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAH, 355a9e644cdSLuigi Rizzo (paddr >> 32) & 0xffffffff); 35646023447SVincenzo Maffione bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAL, 35746023447SVincenzo Maffione paddr & 0xffffffff); 35846023447SVincenzo Maffione paddr = vtophys(sc->csb_hg); 35946023447SVincenzo Maffione bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAH, 36046023447SVincenzo Maffione (paddr >> 32) & 0xffffffff); 36146023447SVincenzo Maffione bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAL, 36246023447SVincenzo Maffione paddr & 0xffffffff); 363a9e644cdSLuigi Rizzo } 364a9e644cdSLuigi Rizzo 365a9e644cdSLuigi Rizzo /* Allocate and initialize per-queue data structures. */ 366ac2fffa4SPedro F. Giffuni sc->queues = malloc(sizeof(struct ptnet_queue) * sc->num_rings, 367a9e644cdSLuigi Rizzo M_DEVBUF, M_NOWAIT | M_ZERO); 368a9e644cdSLuigi Rizzo if (sc->queues == NULL) { 369a9e644cdSLuigi Rizzo err = ENOMEM; 370a9e644cdSLuigi Rizzo goto err_path; 371a9e644cdSLuigi Rizzo } 372a9e644cdSLuigi Rizzo sc->rxqueues = sc->queues + num_tx_rings; 373a9e644cdSLuigi Rizzo 374a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 375a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 376a9e644cdSLuigi Rizzo 377a9e644cdSLuigi Rizzo pq->sc = sc; 378a9e644cdSLuigi Rizzo pq->kring_id = i; 379a9e644cdSLuigi Rizzo pq->kick = PTNET_IO_KICK_BASE + 4 * i; 380b6e66be2SVincenzo Maffione pq->atok = sc->csb_gh + i; 381b6e66be2SVincenzo Maffione pq->ktoa = sc->csb_hg + i; 382a9e644cdSLuigi Rizzo snprintf(pq->lock_name, sizeof(pq->lock_name), "%s-%d", 383a9e644cdSLuigi Rizzo device_get_nameunit(dev), i); 384a9e644cdSLuigi Rizzo mtx_init(&pq->lock, pq->lock_name, NULL, MTX_DEF); 385a9e644cdSLuigi Rizzo if (i >= num_tx_rings) { 386a9e644cdSLuigi Rizzo /* RX queue: fix kring_id. */ 387a9e644cdSLuigi Rizzo pq->kring_id -= num_tx_rings; 388a9e644cdSLuigi Rizzo } else { 389a9e644cdSLuigi Rizzo /* TX queue: allocate buf_ring. */ 390a9e644cdSLuigi Rizzo pq->bufring = buf_ring_alloc(PTNET_BUF_RING_SIZE, 391a9e644cdSLuigi Rizzo M_DEVBUF, M_NOWAIT, &pq->lock); 392a9e644cdSLuigi Rizzo if (pq->bufring == NULL) { 393a9e644cdSLuigi Rizzo err = ENOMEM; 394a9e644cdSLuigi Rizzo goto err_path; 395a9e644cdSLuigi Rizzo } 396a9e644cdSLuigi Rizzo } 397a9e644cdSLuigi Rizzo } 398a9e644cdSLuigi Rizzo 399a9e644cdSLuigi Rizzo sc->min_tx_space = 64; /* Safe initial value. */ 400a9e644cdSLuigi Rizzo 401a9e644cdSLuigi Rizzo err = ptnet_irqs_init(sc); 402a9e644cdSLuigi Rizzo if (err) { 403a9e644cdSLuigi Rizzo goto err_path; 404a9e644cdSLuigi Rizzo } 405a9e644cdSLuigi Rizzo 406a9e644cdSLuigi Rizzo /* Setup Ethernet interface. */ 407a9e644cdSLuigi Rizzo sc->ifp = ifp = if_alloc(IFT_ETHER); 408a9e644cdSLuigi Rizzo if (ifp == NULL) { 409a9e644cdSLuigi Rizzo device_printf(dev, "Failed to allocate ifnet\n"); 410a9e644cdSLuigi Rizzo err = ENOMEM; 411a9e644cdSLuigi Rizzo goto err_path; 412a9e644cdSLuigi Rizzo } 413a9e644cdSLuigi Rizzo 414a9e644cdSLuigi Rizzo if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 415a9e644cdSLuigi Rizzo ifp->if_baudrate = IF_Gbps(10); 416a9e644cdSLuigi Rizzo ifp->if_softc = sc; 417a9e644cdSLuigi Rizzo ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX; 418a9e644cdSLuigi Rizzo ifp->if_init = ptnet_init; 419a9e644cdSLuigi Rizzo ifp->if_ioctl = ptnet_ioctl; 420a9e644cdSLuigi Rizzo #if __FreeBSD_version >= 1100000 421a9e644cdSLuigi Rizzo ifp->if_get_counter = ptnet_get_counter; 422a9e644cdSLuigi Rizzo #endif 423a9e644cdSLuigi Rizzo ifp->if_transmit = ptnet_transmit; 424a9e644cdSLuigi Rizzo ifp->if_qflush = ptnet_qflush; 425a9e644cdSLuigi Rizzo 426a9e644cdSLuigi Rizzo ifmedia_init(&sc->media, IFM_IMASK, ptnet_media_change, 427a9e644cdSLuigi Rizzo ptnet_media_status); 428a9e644cdSLuigi Rizzo ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX, 0, NULL); 429a9e644cdSLuigi Rizzo ifmedia_set(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX); 430a9e644cdSLuigi Rizzo 431a9e644cdSLuigi Rizzo macreg = bus_read_4(sc->iomem, PTNET_IO_MAC_HI); 432a9e644cdSLuigi Rizzo sc->hwaddr[0] = (macreg >> 8) & 0xff; 433a9e644cdSLuigi Rizzo sc->hwaddr[1] = macreg & 0xff; 434a9e644cdSLuigi Rizzo macreg = bus_read_4(sc->iomem, PTNET_IO_MAC_LO); 435a9e644cdSLuigi Rizzo sc->hwaddr[2] = (macreg >> 24) & 0xff; 436a9e644cdSLuigi Rizzo sc->hwaddr[3] = (macreg >> 16) & 0xff; 437a9e644cdSLuigi Rizzo sc->hwaddr[4] = (macreg >> 8) & 0xff; 438a9e644cdSLuigi Rizzo sc->hwaddr[5] = macreg & 0xff; 439a9e644cdSLuigi Rizzo 440a9e644cdSLuigi Rizzo ether_ifattach(ifp, sc->hwaddr); 441a9e644cdSLuigi Rizzo 442a9e644cdSLuigi Rizzo ifp->if_hdrlen = sizeof(struct ether_vlan_header); 443a9e644cdSLuigi Rizzo ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU; 444a9e644cdSLuigi Rizzo 445a9e644cdSLuigi Rizzo if (sc->ptfeatures & PTNETMAP_F_VNET_HDR) { 446a9e644cdSLuigi Rizzo /* Similarly to what the vtnet driver does, we can emulate 447a9e644cdSLuigi Rizzo * VLAN offloadings by inserting and removing the 802.1Q 448a9e644cdSLuigi Rizzo * header during transmit and receive. We are then able 449a9e644cdSLuigi Rizzo * to do checksum offloading of VLAN frames. */ 450a9e644cdSLuigi Rizzo ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 451a9e644cdSLuigi Rizzo | IFCAP_VLAN_HWCSUM 452a9e644cdSLuigi Rizzo | IFCAP_TSO | IFCAP_LRO 453a9e644cdSLuigi Rizzo | IFCAP_VLAN_HWTSO 454a9e644cdSLuigi Rizzo | IFCAP_VLAN_HWTAGGING; 455a9e644cdSLuigi Rizzo } 456a9e644cdSLuigi Rizzo 457a9e644cdSLuigi Rizzo ifp->if_capenable = ifp->if_capabilities; 458a9e644cdSLuigi Rizzo #ifdef DEVICE_POLLING 459a9e644cdSLuigi Rizzo /* Don't enable polling by default. */ 460a9e644cdSLuigi Rizzo ifp->if_capabilities |= IFCAP_POLLING; 461a9e644cdSLuigi Rizzo #endif 462a9e644cdSLuigi Rizzo snprintf(sc->lock_name, sizeof(sc->lock_name), 463a9e644cdSLuigi Rizzo "%s", device_get_nameunit(dev)); 464a9e644cdSLuigi Rizzo mtx_init(&sc->lock, sc->lock_name, "ptnet core lock", MTX_DEF); 465a9e644cdSLuigi Rizzo callout_init_mtx(&sc->tick, &sc->lock, 0); 466a9e644cdSLuigi Rizzo 467a9e644cdSLuigi Rizzo /* Prepare a netmap_adapter struct instance to do netmap_attach(). */ 468a9e644cdSLuigi Rizzo nifp_offset = bus_read_4(sc->iomem, PTNET_IO_NIFP_OFS); 469a9e644cdSLuigi Rizzo memset(&na_arg, 0, sizeof(na_arg)); 470a9e644cdSLuigi Rizzo na_arg.ifp = ifp; 471a9e644cdSLuigi Rizzo na_arg.num_tx_desc = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_SLOTS); 472a9e644cdSLuigi Rizzo na_arg.num_rx_desc = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_SLOTS); 473a9e644cdSLuigi Rizzo na_arg.num_tx_rings = num_tx_rings; 474a9e644cdSLuigi Rizzo na_arg.num_rx_rings = num_rx_rings; 475a9e644cdSLuigi Rizzo na_arg.nm_config = ptnet_nm_config; 476a9e644cdSLuigi Rizzo na_arg.nm_krings_create = ptnet_nm_krings_create; 477a9e644cdSLuigi Rizzo na_arg.nm_krings_delete = ptnet_nm_krings_delete; 478a9e644cdSLuigi Rizzo na_arg.nm_dtor = ptnet_nm_dtor; 4794f80b14cSVincenzo Maffione na_arg.nm_intr = ptnet_nm_intr; 480a9e644cdSLuigi Rizzo na_arg.nm_register = ptnet_nm_register; 481a9e644cdSLuigi Rizzo na_arg.nm_txsync = ptnet_nm_txsync; 482a9e644cdSLuigi Rizzo na_arg.nm_rxsync = ptnet_nm_rxsync; 483a9e644cdSLuigi Rizzo 48446023447SVincenzo Maffione netmap_pt_guest_attach(&na_arg, nifp_offset, 485844a6f0cSLuigi Rizzo bus_read_4(sc->iomem, PTNET_IO_HOSTMEMID)); 486a9e644cdSLuigi Rizzo 487a9e644cdSLuigi Rizzo /* Now a netmap adapter for this ifp has been allocated, and it 488a9e644cdSLuigi Rizzo * can be accessed through NA(ifp). We also have to initialize the CSB 489a9e644cdSLuigi Rizzo * pointer. */ 490a9e644cdSLuigi Rizzo sc->ptna = (struct netmap_pt_guest_adapter *)NA(ifp); 491a9e644cdSLuigi Rizzo 492a9e644cdSLuigi Rizzo /* If virtio-net header was negotiated, set the virt_hdr_len field in 493a9e644cdSLuigi Rizzo * the netmap adapter, to inform users that this netmap adapter requires 494a9e644cdSLuigi Rizzo * the application to deal with the headers. */ 495a9e644cdSLuigi Rizzo ptnet_update_vnet_hdr(sc); 496a9e644cdSLuigi Rizzo 497a9e644cdSLuigi Rizzo device_printf(dev, "%s() completed\n", __func__); 498a9e644cdSLuigi Rizzo 499a9e644cdSLuigi Rizzo return (0); 500a9e644cdSLuigi Rizzo 501a9e644cdSLuigi Rizzo err_path: 502a9e644cdSLuigi Rizzo ptnet_detach(dev); 503a9e644cdSLuigi Rizzo return err; 504a9e644cdSLuigi Rizzo } 505a9e644cdSLuigi Rizzo 506b6e66be2SVincenzo Maffione /* Stop host sync-kloop if it was running. */ 507b6e66be2SVincenzo Maffione static void 508b6e66be2SVincenzo Maffione ptnet_device_shutdown(struct ptnet_softc *sc) 509b6e66be2SVincenzo Maffione { 510b6e66be2SVincenzo Maffione ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_DELETE); 511b6e66be2SVincenzo Maffione bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAH, 0); 512b6e66be2SVincenzo Maffione bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAL, 0); 513b6e66be2SVincenzo Maffione bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAH, 0); 514b6e66be2SVincenzo Maffione bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAL, 0); 515b6e66be2SVincenzo Maffione } 516b6e66be2SVincenzo Maffione 517a9e644cdSLuigi Rizzo static int 518a9e644cdSLuigi Rizzo ptnet_detach(device_t dev) 519a9e644cdSLuigi Rizzo { 520a9e644cdSLuigi Rizzo struct ptnet_softc *sc = device_get_softc(dev); 521a9e644cdSLuigi Rizzo int i; 522a9e644cdSLuigi Rizzo 523b6e66be2SVincenzo Maffione ptnet_device_shutdown(sc); 524b6e66be2SVincenzo Maffione 525a9e644cdSLuigi Rizzo #ifdef DEVICE_POLLING 526a9e644cdSLuigi Rizzo if (sc->ifp->if_capenable & IFCAP_POLLING) { 527a9e644cdSLuigi Rizzo ether_poll_deregister(sc->ifp); 528a9e644cdSLuigi Rizzo } 529a9e644cdSLuigi Rizzo #endif 530a9e644cdSLuigi Rizzo callout_drain(&sc->tick); 531a9e644cdSLuigi Rizzo 532a9e644cdSLuigi Rizzo if (sc->queues) { 533a9e644cdSLuigi Rizzo /* Drain taskqueues before calling if_detach. */ 534a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 535a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 536a9e644cdSLuigi Rizzo 537a9e644cdSLuigi Rizzo if (pq->taskq) { 538a9e644cdSLuigi Rizzo taskqueue_drain(pq->taskq, &pq->task); 539a9e644cdSLuigi Rizzo } 540a9e644cdSLuigi Rizzo } 541a9e644cdSLuigi Rizzo } 542a9e644cdSLuigi Rizzo 543a9e644cdSLuigi Rizzo if (sc->ifp) { 544a9e644cdSLuigi Rizzo ether_ifdetach(sc->ifp); 545a9e644cdSLuigi Rizzo 546a9e644cdSLuigi Rizzo /* Uninitialize netmap adapters for this device. */ 547a9e644cdSLuigi Rizzo netmap_detach(sc->ifp); 548a9e644cdSLuigi Rizzo 549a9e644cdSLuigi Rizzo ifmedia_removeall(&sc->media); 550a9e644cdSLuigi Rizzo if_free(sc->ifp); 551a9e644cdSLuigi Rizzo sc->ifp = NULL; 552a9e644cdSLuigi Rizzo } 553a9e644cdSLuigi Rizzo 554a9e644cdSLuigi Rizzo ptnet_irqs_fini(sc); 555a9e644cdSLuigi Rizzo 55646023447SVincenzo Maffione if (sc->csb_gh) { 55746023447SVincenzo Maffione contigfree(sc->csb_gh, 2*PAGE_SIZE, M_DEVBUF); 55846023447SVincenzo Maffione sc->csb_gh = NULL; 55946023447SVincenzo Maffione sc->csb_hg = NULL; 560a9e644cdSLuigi Rizzo } 561a9e644cdSLuigi Rizzo 562a9e644cdSLuigi Rizzo if (sc->queues) { 563a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 564a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 565a9e644cdSLuigi Rizzo 566a9e644cdSLuigi Rizzo if (mtx_initialized(&pq->lock)) { 567a9e644cdSLuigi Rizzo mtx_destroy(&pq->lock); 568a9e644cdSLuigi Rizzo } 569a9e644cdSLuigi Rizzo if (pq->bufring != NULL) { 570a9e644cdSLuigi Rizzo buf_ring_free(pq->bufring, M_DEVBUF); 571a9e644cdSLuigi Rizzo } 572a9e644cdSLuigi Rizzo } 573a9e644cdSLuigi Rizzo free(sc->queues, M_DEVBUF); 574a9e644cdSLuigi Rizzo sc->queues = NULL; 575a9e644cdSLuigi Rizzo } 576a9e644cdSLuigi Rizzo 577a9e644cdSLuigi Rizzo if (sc->iomem) { 578a9e644cdSLuigi Rizzo bus_release_resource(dev, SYS_RES_IOPORT, 579a9e644cdSLuigi Rizzo PCIR_BAR(PTNETMAP_IO_PCI_BAR), sc->iomem); 580a9e644cdSLuigi Rizzo sc->iomem = NULL; 581a9e644cdSLuigi Rizzo } 582a9e644cdSLuigi Rizzo 583a9e644cdSLuigi Rizzo mtx_destroy(&sc->lock); 584a9e644cdSLuigi Rizzo 585a9e644cdSLuigi Rizzo device_printf(dev, "%s() completed\n", __func__); 586a9e644cdSLuigi Rizzo 587a9e644cdSLuigi Rizzo return (0); 588a9e644cdSLuigi Rizzo } 589a9e644cdSLuigi Rizzo 590a9e644cdSLuigi Rizzo static int 591a9e644cdSLuigi Rizzo ptnet_suspend(device_t dev) 592a9e644cdSLuigi Rizzo { 593b6e66be2SVincenzo Maffione struct ptnet_softc *sc = device_get_softc(dev); 594a9e644cdSLuigi Rizzo 595a9e644cdSLuigi Rizzo (void)sc; 596a9e644cdSLuigi Rizzo 597a9e644cdSLuigi Rizzo return (0); 598a9e644cdSLuigi Rizzo } 599a9e644cdSLuigi Rizzo 600a9e644cdSLuigi Rizzo static int 601a9e644cdSLuigi Rizzo ptnet_resume(device_t dev) 602a9e644cdSLuigi Rizzo { 603b6e66be2SVincenzo Maffione struct ptnet_softc *sc = device_get_softc(dev); 604a9e644cdSLuigi Rizzo 605a9e644cdSLuigi Rizzo (void)sc; 606a9e644cdSLuigi Rizzo 607a9e644cdSLuigi Rizzo return (0); 608a9e644cdSLuigi Rizzo } 609a9e644cdSLuigi Rizzo 610a9e644cdSLuigi Rizzo static int 611a9e644cdSLuigi Rizzo ptnet_shutdown(device_t dev) 612a9e644cdSLuigi Rizzo { 613b6e66be2SVincenzo Maffione struct ptnet_softc *sc = device_get_softc(dev); 614b6e66be2SVincenzo Maffione 615b6e66be2SVincenzo Maffione ptnet_device_shutdown(sc); 616b6e66be2SVincenzo Maffione 617b6e66be2SVincenzo Maffione return (0); 618a9e644cdSLuigi Rizzo } 619a9e644cdSLuigi Rizzo 620a9e644cdSLuigi Rizzo static int 621a9e644cdSLuigi Rizzo ptnet_irqs_init(struct ptnet_softc *sc) 622a9e644cdSLuigi Rizzo { 623a9e644cdSLuigi Rizzo int rid = PCIR_BAR(PTNETMAP_MSIX_PCI_BAR); 624a9e644cdSLuigi Rizzo int nvecs = sc->num_rings; 625a9e644cdSLuigi Rizzo device_t dev = sc->dev; 626a9e644cdSLuigi Rizzo int err = ENOSPC; 627a9e644cdSLuigi Rizzo int cpu_cur; 628a9e644cdSLuigi Rizzo int i; 629a9e644cdSLuigi Rizzo 630a9e644cdSLuigi Rizzo if (pci_find_cap(dev, PCIY_MSIX, NULL) != 0) { 631a9e644cdSLuigi Rizzo device_printf(dev, "Could not find MSI-X capability\n"); 632a9e644cdSLuigi Rizzo return (ENXIO); 633a9e644cdSLuigi Rizzo } 634a9e644cdSLuigi Rizzo 635a9e644cdSLuigi Rizzo sc->msix_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 636a9e644cdSLuigi Rizzo &rid, RF_ACTIVE); 637a9e644cdSLuigi Rizzo if (sc->msix_mem == NULL) { 638a9e644cdSLuigi Rizzo device_printf(dev, "Failed to allocate MSIX PCI BAR\n"); 639a9e644cdSLuigi Rizzo return (ENXIO); 640a9e644cdSLuigi Rizzo } 641a9e644cdSLuigi Rizzo 642a9e644cdSLuigi Rizzo if (pci_msix_count(dev) < nvecs) { 643a9e644cdSLuigi Rizzo device_printf(dev, "Not enough MSI-X vectors\n"); 644a9e644cdSLuigi Rizzo goto err_path; 645a9e644cdSLuigi Rizzo } 646a9e644cdSLuigi Rizzo 647a9e644cdSLuigi Rizzo err = pci_alloc_msix(dev, &nvecs); 648a9e644cdSLuigi Rizzo if (err) { 649a9e644cdSLuigi Rizzo device_printf(dev, "Failed to allocate MSI-X vectors\n"); 650a9e644cdSLuigi Rizzo goto err_path; 651a9e644cdSLuigi Rizzo } 652a9e644cdSLuigi Rizzo 653a9e644cdSLuigi Rizzo for (i = 0; i < nvecs; i++) { 654a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 655a9e644cdSLuigi Rizzo 656a9e644cdSLuigi Rizzo rid = i + 1; 657a9e644cdSLuigi Rizzo pq->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 658a9e644cdSLuigi Rizzo RF_ACTIVE); 659a9e644cdSLuigi Rizzo if (pq->irq == NULL) { 660a9e644cdSLuigi Rizzo device_printf(dev, "Failed to allocate interrupt " 661a9e644cdSLuigi Rizzo "for queue #%d\n", i); 662a9e644cdSLuigi Rizzo err = ENOSPC; 663a9e644cdSLuigi Rizzo goto err_path; 664a9e644cdSLuigi Rizzo } 665a9e644cdSLuigi Rizzo } 666a9e644cdSLuigi Rizzo 667a9e644cdSLuigi Rizzo cpu_cur = CPU_FIRST(); 668a9e644cdSLuigi Rizzo for (i = 0; i < nvecs; i++) { 669a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 670a9e644cdSLuigi Rizzo void (*handler)(void *) = ptnet_tx_intr; 671a9e644cdSLuigi Rizzo 672a9e644cdSLuigi Rizzo if (i >= sc->num_tx_rings) { 673a9e644cdSLuigi Rizzo handler = ptnet_rx_intr; 674a9e644cdSLuigi Rizzo } 675a9e644cdSLuigi Rizzo err = bus_setup_intr(dev, pq->irq, INTR_TYPE_NET | INTR_MPSAFE, 676a9e644cdSLuigi Rizzo NULL /* intr_filter */, handler, 677a9e644cdSLuigi Rizzo pq, &pq->cookie); 678a9e644cdSLuigi Rizzo if (err) { 679a9e644cdSLuigi Rizzo device_printf(dev, "Failed to register intr handler " 680a9e644cdSLuigi Rizzo "for queue #%d\n", i); 681a9e644cdSLuigi Rizzo goto err_path; 682a9e644cdSLuigi Rizzo } 683a9e644cdSLuigi Rizzo 684a9e644cdSLuigi Rizzo bus_describe_intr(dev, pq->irq, pq->cookie, "q%d", i); 685a9e644cdSLuigi Rizzo #if 0 686a9e644cdSLuigi Rizzo bus_bind_intr(sc->dev, pq->irq, cpu_cur); 687a9e644cdSLuigi Rizzo #endif 688a9e644cdSLuigi Rizzo cpu_cur = CPU_NEXT(cpu_cur); 689a9e644cdSLuigi Rizzo } 690a9e644cdSLuigi Rizzo 691a9e644cdSLuigi Rizzo device_printf(dev, "Allocated %d MSI-X vectors\n", nvecs); 692a9e644cdSLuigi Rizzo 693a9e644cdSLuigi Rizzo cpu_cur = CPU_FIRST(); 694a9e644cdSLuigi Rizzo for (i = 0; i < nvecs; i++) { 695a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 696a9e644cdSLuigi Rizzo static void (*handler)(void *context, int pending); 697a9e644cdSLuigi Rizzo 698a9e644cdSLuigi Rizzo handler = (i < sc->num_tx_rings) ? ptnet_tx_task : ptnet_rx_task; 699a9e644cdSLuigi Rizzo 700a9e644cdSLuigi Rizzo TASK_INIT(&pq->task, 0, handler, pq); 701a9e644cdSLuigi Rizzo pq->taskq = taskqueue_create_fast("ptnet_queue", M_NOWAIT, 702a9e644cdSLuigi Rizzo taskqueue_thread_enqueue, &pq->taskq); 703a9e644cdSLuigi Rizzo taskqueue_start_threads(&pq->taskq, 1, PI_NET, "%s-pq-%d", 704a9e644cdSLuigi Rizzo device_get_nameunit(sc->dev), cpu_cur); 705a9e644cdSLuigi Rizzo cpu_cur = CPU_NEXT(cpu_cur); 706a9e644cdSLuigi Rizzo } 707a9e644cdSLuigi Rizzo 708a9e644cdSLuigi Rizzo return 0; 709a9e644cdSLuigi Rizzo err_path: 710a9e644cdSLuigi Rizzo ptnet_irqs_fini(sc); 711a9e644cdSLuigi Rizzo return err; 712a9e644cdSLuigi Rizzo } 713a9e644cdSLuigi Rizzo 714a9e644cdSLuigi Rizzo static void 715a9e644cdSLuigi Rizzo ptnet_irqs_fini(struct ptnet_softc *sc) 716a9e644cdSLuigi Rizzo { 717a9e644cdSLuigi Rizzo device_t dev = sc->dev; 718a9e644cdSLuigi Rizzo int i; 719a9e644cdSLuigi Rizzo 720a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 721a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 722a9e644cdSLuigi Rizzo 723a9e644cdSLuigi Rizzo if (pq->taskq) { 724a9e644cdSLuigi Rizzo taskqueue_free(pq->taskq); 725a9e644cdSLuigi Rizzo pq->taskq = NULL; 726a9e644cdSLuigi Rizzo } 727a9e644cdSLuigi Rizzo 728a9e644cdSLuigi Rizzo if (pq->cookie) { 729a9e644cdSLuigi Rizzo bus_teardown_intr(dev, pq->irq, pq->cookie); 730a9e644cdSLuigi Rizzo pq->cookie = NULL; 731a9e644cdSLuigi Rizzo } 732a9e644cdSLuigi Rizzo 733a9e644cdSLuigi Rizzo if (pq->irq) { 734a9e644cdSLuigi Rizzo bus_release_resource(dev, SYS_RES_IRQ, i + 1, pq->irq); 735a9e644cdSLuigi Rizzo pq->irq = NULL; 736a9e644cdSLuigi Rizzo } 737a9e644cdSLuigi Rizzo } 738a9e644cdSLuigi Rizzo 739a9e644cdSLuigi Rizzo if (sc->msix_mem) { 740a9e644cdSLuigi Rizzo pci_release_msi(dev); 741a9e644cdSLuigi Rizzo 742a9e644cdSLuigi Rizzo bus_release_resource(dev, SYS_RES_MEMORY, 743a9e644cdSLuigi Rizzo PCIR_BAR(PTNETMAP_MSIX_PCI_BAR), 744a9e644cdSLuigi Rizzo sc->msix_mem); 745a9e644cdSLuigi Rizzo sc->msix_mem = NULL; 746a9e644cdSLuigi Rizzo } 747a9e644cdSLuigi Rizzo } 748a9e644cdSLuigi Rizzo 749a9e644cdSLuigi Rizzo static void 750a9e644cdSLuigi Rizzo ptnet_init(void *opaque) 751a9e644cdSLuigi Rizzo { 752a9e644cdSLuigi Rizzo struct ptnet_softc *sc = opaque; 753a9e644cdSLuigi Rizzo 754a9e644cdSLuigi Rizzo PTNET_CORE_LOCK(sc); 755a9e644cdSLuigi Rizzo ptnet_init_locked(sc); 756a9e644cdSLuigi Rizzo PTNET_CORE_UNLOCK(sc); 757a9e644cdSLuigi Rizzo } 758a9e644cdSLuigi Rizzo 759a9e644cdSLuigi Rizzo static int 760a9e644cdSLuigi Rizzo ptnet_ioctl(if_t ifp, u_long cmd, caddr_t data) 761a9e644cdSLuigi Rizzo { 762a9e644cdSLuigi Rizzo struct ptnet_softc *sc = if_getsoftc(ifp); 763a9e644cdSLuigi Rizzo device_t dev = sc->dev; 764a9e644cdSLuigi Rizzo struct ifreq *ifr = (struct ifreq *)data; 76524a7d6d3SMatt Macy int mask __unused, err = 0; 766a9e644cdSLuigi Rizzo 767a9e644cdSLuigi Rizzo switch (cmd) { 768a9e644cdSLuigi Rizzo case SIOCSIFFLAGS: 769a9e644cdSLuigi Rizzo device_printf(dev, "SIOCSIFFLAGS %x\n", ifp->if_flags); 770a9e644cdSLuigi Rizzo PTNET_CORE_LOCK(sc); 771a9e644cdSLuigi Rizzo if (ifp->if_flags & IFF_UP) { 772a9e644cdSLuigi Rizzo /* Network stack wants the iff to be up. */ 773a9e644cdSLuigi Rizzo err = ptnet_init_locked(sc); 774a9e644cdSLuigi Rizzo } else { 775a9e644cdSLuigi Rizzo /* Network stack wants the iff to be down. */ 776a9e644cdSLuigi Rizzo err = ptnet_stop(sc); 777a9e644cdSLuigi Rizzo } 778a9e644cdSLuigi Rizzo /* We don't need to do nothing to support IFF_PROMISC, 779a9e644cdSLuigi Rizzo * since that is managed by the backend port. */ 780a9e644cdSLuigi Rizzo PTNET_CORE_UNLOCK(sc); 781a9e644cdSLuigi Rizzo break; 782a9e644cdSLuigi Rizzo 783a9e644cdSLuigi Rizzo case SIOCSIFCAP: 784a9e644cdSLuigi Rizzo device_printf(dev, "SIOCSIFCAP %x %x\n", 785a9e644cdSLuigi Rizzo ifr->ifr_reqcap, ifp->if_capenable); 786a9e644cdSLuigi Rizzo mask = ifr->ifr_reqcap ^ ifp->if_capenable; 787a9e644cdSLuigi Rizzo #ifdef DEVICE_POLLING 788a9e644cdSLuigi Rizzo if (mask & IFCAP_POLLING) { 789a9e644cdSLuigi Rizzo struct ptnet_queue *pq; 790a9e644cdSLuigi Rizzo int i; 791a9e644cdSLuigi Rizzo 792a9e644cdSLuigi Rizzo if (ifr->ifr_reqcap & IFCAP_POLLING) { 793a9e644cdSLuigi Rizzo err = ether_poll_register(ptnet_poll, ifp); 794a9e644cdSLuigi Rizzo if (err) { 795a9e644cdSLuigi Rizzo break; 796a9e644cdSLuigi Rizzo } 797a9e644cdSLuigi Rizzo /* Stop queues and sync with taskqueues. */ 798a9e644cdSLuigi Rizzo ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 799a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 800a9e644cdSLuigi Rizzo pq = sc-> queues + i; 801a9e644cdSLuigi Rizzo /* Make sure the worker sees the 802a9e644cdSLuigi Rizzo * IFF_DRV_RUNNING down. */ 803a9e644cdSLuigi Rizzo PTNET_Q_LOCK(pq); 804b6e66be2SVincenzo Maffione pq->atok->appl_need_kick = 0; 805a9e644cdSLuigi Rizzo PTNET_Q_UNLOCK(pq); 806a9e644cdSLuigi Rizzo /* Wait for rescheduling to finish. */ 807a9e644cdSLuigi Rizzo if (pq->taskq) { 808a9e644cdSLuigi Rizzo taskqueue_drain(pq->taskq, 809a9e644cdSLuigi Rizzo &pq->task); 810a9e644cdSLuigi Rizzo } 811a9e644cdSLuigi Rizzo } 812a9e644cdSLuigi Rizzo ifp->if_drv_flags |= IFF_DRV_RUNNING; 813a9e644cdSLuigi Rizzo } else { 814a9e644cdSLuigi Rizzo err = ether_poll_deregister(ifp); 815a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 816a9e644cdSLuigi Rizzo pq = sc-> queues + i; 817a9e644cdSLuigi Rizzo PTNET_Q_LOCK(pq); 818b6e66be2SVincenzo Maffione pq->atok->appl_need_kick = 1; 819a9e644cdSLuigi Rizzo PTNET_Q_UNLOCK(pq); 820a9e644cdSLuigi Rizzo } 821a9e644cdSLuigi Rizzo } 822a9e644cdSLuigi Rizzo } 823a9e644cdSLuigi Rizzo #endif /* DEVICE_POLLING */ 824a9e644cdSLuigi Rizzo ifp->if_capenable = ifr->ifr_reqcap; 825a9e644cdSLuigi Rizzo break; 826a9e644cdSLuigi Rizzo 827a9e644cdSLuigi Rizzo case SIOCSIFMTU: 828a9e644cdSLuigi Rizzo /* We support any reasonable MTU. */ 829a9e644cdSLuigi Rizzo if (ifr->ifr_mtu < ETHERMIN || 830a9e644cdSLuigi Rizzo ifr->ifr_mtu > PTNET_MAX_PKT_SIZE) { 831a9e644cdSLuigi Rizzo err = EINVAL; 832a9e644cdSLuigi Rizzo } else { 833a9e644cdSLuigi Rizzo PTNET_CORE_LOCK(sc); 834a9e644cdSLuigi Rizzo ifp->if_mtu = ifr->ifr_mtu; 835a9e644cdSLuigi Rizzo PTNET_CORE_UNLOCK(sc); 836a9e644cdSLuigi Rizzo } 837a9e644cdSLuigi Rizzo break; 838a9e644cdSLuigi Rizzo 839a9e644cdSLuigi Rizzo case SIOCSIFMEDIA: 840a9e644cdSLuigi Rizzo case SIOCGIFMEDIA: 841a9e644cdSLuigi Rizzo err = ifmedia_ioctl(ifp, ifr, &sc->media, cmd); 842a9e644cdSLuigi Rizzo break; 843a9e644cdSLuigi Rizzo 844a9e644cdSLuigi Rizzo default: 845a9e644cdSLuigi Rizzo err = ether_ioctl(ifp, cmd, data); 846a9e644cdSLuigi Rizzo break; 847a9e644cdSLuigi Rizzo } 848a9e644cdSLuigi Rizzo 849a9e644cdSLuigi Rizzo return err; 850a9e644cdSLuigi Rizzo } 851a9e644cdSLuigi Rizzo 852a9e644cdSLuigi Rizzo static int 853a9e644cdSLuigi Rizzo ptnet_init_locked(struct ptnet_softc *sc) 854a9e644cdSLuigi Rizzo { 855a9e644cdSLuigi Rizzo if_t ifp = sc->ifp; 856a9e644cdSLuigi Rizzo struct netmap_adapter *na_dr = &sc->ptna->dr.up; 857a9e644cdSLuigi Rizzo struct netmap_adapter *na_nm = &sc->ptna->hwup.up; 858a9e644cdSLuigi Rizzo unsigned int nm_buf_size; 859a9e644cdSLuigi Rizzo int ret; 860a9e644cdSLuigi Rizzo 861a9e644cdSLuigi Rizzo if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 862a9e644cdSLuigi Rizzo return 0; /* nothing to do */ 863a9e644cdSLuigi Rizzo } 864a9e644cdSLuigi Rizzo 865a9e644cdSLuigi Rizzo device_printf(sc->dev, "%s\n", __func__); 866a9e644cdSLuigi Rizzo 867a9e644cdSLuigi Rizzo /* Translate offload capabilities according to if_capenable. */ 868a9e644cdSLuigi Rizzo ifp->if_hwassist = 0; 869a9e644cdSLuigi Rizzo if (ifp->if_capenable & IFCAP_TXCSUM) 870a9e644cdSLuigi Rizzo ifp->if_hwassist |= PTNET_CSUM_OFFLOAD; 871a9e644cdSLuigi Rizzo if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) 872a9e644cdSLuigi Rizzo ifp->if_hwassist |= PTNET_CSUM_OFFLOAD_IPV6; 873a9e644cdSLuigi Rizzo if (ifp->if_capenable & IFCAP_TSO4) 874a9e644cdSLuigi Rizzo ifp->if_hwassist |= CSUM_IP_TSO; 875a9e644cdSLuigi Rizzo if (ifp->if_capenable & IFCAP_TSO6) 876a9e644cdSLuigi Rizzo ifp->if_hwassist |= CSUM_IP6_TSO; 877a9e644cdSLuigi Rizzo 878a9e644cdSLuigi Rizzo /* 879a9e644cdSLuigi Rizzo * Prepare the interface for netmap mode access. 880a9e644cdSLuigi Rizzo */ 881a9e644cdSLuigi Rizzo netmap_update_config(na_dr); 882a9e644cdSLuigi Rizzo 883a9e644cdSLuigi Rizzo ret = netmap_mem_finalize(na_dr->nm_mem, na_dr); 884a9e644cdSLuigi Rizzo if (ret) { 885a9e644cdSLuigi Rizzo device_printf(sc->dev, "netmap_mem_finalize() failed\n"); 886a9e644cdSLuigi Rizzo return ret; 887a9e644cdSLuigi Rizzo } 888a9e644cdSLuigi Rizzo 889b6e66be2SVincenzo Maffione if (sc->ptna->backend_users == 0) { 890a9e644cdSLuigi Rizzo ret = ptnet_nm_krings_create(na_nm); 891a9e644cdSLuigi Rizzo if (ret) { 892a9e644cdSLuigi Rizzo device_printf(sc->dev, "ptnet_nm_krings_create() " 893a9e644cdSLuigi Rizzo "failed\n"); 894a9e644cdSLuigi Rizzo goto err_mem_finalize; 895a9e644cdSLuigi Rizzo } 896a9e644cdSLuigi Rizzo 897a9e644cdSLuigi Rizzo ret = netmap_mem_rings_create(na_dr); 898a9e644cdSLuigi Rizzo if (ret) { 899a9e644cdSLuigi Rizzo device_printf(sc->dev, "netmap_mem_rings_create() " 900a9e644cdSLuigi Rizzo "failed\n"); 901a9e644cdSLuigi Rizzo goto err_rings_create; 902a9e644cdSLuigi Rizzo } 903a9e644cdSLuigi Rizzo 904a9e644cdSLuigi Rizzo ret = netmap_mem_get_lut(na_dr->nm_mem, &na_dr->na_lut); 905a9e644cdSLuigi Rizzo if (ret) { 906a9e644cdSLuigi Rizzo device_printf(sc->dev, "netmap_mem_get_lut() " 907a9e644cdSLuigi Rizzo "failed\n"); 908a9e644cdSLuigi Rizzo goto err_get_lut; 909a9e644cdSLuigi Rizzo } 910a9e644cdSLuigi Rizzo } 911a9e644cdSLuigi Rizzo 912a9e644cdSLuigi Rizzo ret = ptnet_nm_register(na_dr, 1 /* on */); 913a9e644cdSLuigi Rizzo if (ret) { 914a9e644cdSLuigi Rizzo goto err_register; 915a9e644cdSLuigi Rizzo } 916a9e644cdSLuigi Rizzo 917a9e644cdSLuigi Rizzo nm_buf_size = NETMAP_BUF_SIZE(na_dr); 918a9e644cdSLuigi Rizzo 919a9e644cdSLuigi Rizzo KASSERT(nm_buf_size > 0, ("Invalid netmap buffer size")); 920a9e644cdSLuigi Rizzo sc->min_tx_space = PTNET_MAX_PKT_SIZE / nm_buf_size + 2; 921a9e644cdSLuigi Rizzo device_printf(sc->dev, "%s: min_tx_space = %u\n", __func__, 922a9e644cdSLuigi Rizzo sc->min_tx_space); 923a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS 924a9e644cdSLuigi Rizzo callout_reset(&sc->tick, hz, ptnet_tick, sc); 925a9e644cdSLuigi Rizzo #endif 926a9e644cdSLuigi Rizzo 927a9e644cdSLuigi Rizzo ifp->if_drv_flags |= IFF_DRV_RUNNING; 928a9e644cdSLuigi Rizzo 929a9e644cdSLuigi Rizzo return 0; 930a9e644cdSLuigi Rizzo 931a9e644cdSLuigi Rizzo err_register: 932a9e644cdSLuigi Rizzo memset(&na_dr->na_lut, 0, sizeof(na_dr->na_lut)); 933a9e644cdSLuigi Rizzo err_get_lut: 934a9e644cdSLuigi Rizzo netmap_mem_rings_delete(na_dr); 935a9e644cdSLuigi Rizzo err_rings_create: 936a9e644cdSLuigi Rizzo ptnet_nm_krings_delete(na_nm); 937a9e644cdSLuigi Rizzo err_mem_finalize: 938a9e644cdSLuigi Rizzo netmap_mem_deref(na_dr->nm_mem, na_dr); 939a9e644cdSLuigi Rizzo 940a9e644cdSLuigi Rizzo return ret; 941a9e644cdSLuigi Rizzo } 942a9e644cdSLuigi Rizzo 943a9e644cdSLuigi Rizzo /* To be called under core lock. */ 944a9e644cdSLuigi Rizzo static int 945a9e644cdSLuigi Rizzo ptnet_stop(struct ptnet_softc *sc) 946a9e644cdSLuigi Rizzo { 947a9e644cdSLuigi Rizzo if_t ifp = sc->ifp; 948a9e644cdSLuigi Rizzo struct netmap_adapter *na_dr = &sc->ptna->dr.up; 949a9e644cdSLuigi Rizzo struct netmap_adapter *na_nm = &sc->ptna->hwup.up; 950a9e644cdSLuigi Rizzo int i; 951a9e644cdSLuigi Rizzo 952a9e644cdSLuigi Rizzo device_printf(sc->dev, "%s\n", __func__); 953a9e644cdSLuigi Rizzo 954a9e644cdSLuigi Rizzo if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 955a9e644cdSLuigi Rizzo return 0; /* nothing to do */ 956a9e644cdSLuigi Rizzo } 957a9e644cdSLuigi Rizzo 958a9e644cdSLuigi Rizzo /* Clear the driver-ready flag, and synchronize with all the queues, 959a9e644cdSLuigi Rizzo * so that after this loop we are sure nobody is working anymore with 960a9e644cdSLuigi Rizzo * the device. This scheme is taken from the vtnet driver. */ 961a9e644cdSLuigi Rizzo ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 962a9e644cdSLuigi Rizzo callout_stop(&sc->tick); 963a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 964a9e644cdSLuigi Rizzo PTNET_Q_LOCK(sc->queues + i); 965a9e644cdSLuigi Rizzo PTNET_Q_UNLOCK(sc->queues + i); 966a9e644cdSLuigi Rizzo } 967a9e644cdSLuigi Rizzo 968a9e644cdSLuigi Rizzo ptnet_nm_register(na_dr, 0 /* off */); 969a9e644cdSLuigi Rizzo 970b6e66be2SVincenzo Maffione if (sc->ptna->backend_users == 0) { 971a9e644cdSLuigi Rizzo netmap_mem_rings_delete(na_dr); 972a9e644cdSLuigi Rizzo ptnet_nm_krings_delete(na_nm); 973a9e644cdSLuigi Rizzo } 974a9e644cdSLuigi Rizzo netmap_mem_deref(na_dr->nm_mem, na_dr); 975a9e644cdSLuigi Rizzo 976a9e644cdSLuigi Rizzo return 0; 977a9e644cdSLuigi Rizzo } 978a9e644cdSLuigi Rizzo 979a9e644cdSLuigi Rizzo static void 980a9e644cdSLuigi Rizzo ptnet_qflush(if_t ifp) 981a9e644cdSLuigi Rizzo { 982a9e644cdSLuigi Rizzo struct ptnet_softc *sc = if_getsoftc(ifp); 983a9e644cdSLuigi Rizzo int i; 984a9e644cdSLuigi Rizzo 985a9e644cdSLuigi Rizzo /* Flush all the bufrings and do the interface flush. */ 986a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 987a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 988a9e644cdSLuigi Rizzo struct mbuf *m; 989a9e644cdSLuigi Rizzo 990a9e644cdSLuigi Rizzo PTNET_Q_LOCK(pq); 991a9e644cdSLuigi Rizzo if (pq->bufring) { 992a9e644cdSLuigi Rizzo while ((m = buf_ring_dequeue_sc(pq->bufring))) { 993a9e644cdSLuigi Rizzo m_freem(m); 994a9e644cdSLuigi Rizzo } 995a9e644cdSLuigi Rizzo } 996a9e644cdSLuigi Rizzo PTNET_Q_UNLOCK(pq); 997a9e644cdSLuigi Rizzo } 998a9e644cdSLuigi Rizzo 999a9e644cdSLuigi Rizzo if_qflush(ifp); 1000a9e644cdSLuigi Rizzo } 1001a9e644cdSLuigi Rizzo 1002a9e644cdSLuigi Rizzo static int 1003a9e644cdSLuigi Rizzo ptnet_media_change(if_t ifp) 1004a9e644cdSLuigi Rizzo { 1005a9e644cdSLuigi Rizzo struct ptnet_softc *sc = if_getsoftc(ifp); 1006a9e644cdSLuigi Rizzo struct ifmedia *ifm = &sc->media; 1007a9e644cdSLuigi Rizzo 1008a9e644cdSLuigi Rizzo if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) { 1009a9e644cdSLuigi Rizzo return EINVAL; 1010a9e644cdSLuigi Rizzo } 1011a9e644cdSLuigi Rizzo 1012a9e644cdSLuigi Rizzo return 0; 1013a9e644cdSLuigi Rizzo } 1014a9e644cdSLuigi Rizzo 1015a9e644cdSLuigi Rizzo #if __FreeBSD_version >= 1100000 1016a9e644cdSLuigi Rizzo static uint64_t 1017a9e644cdSLuigi Rizzo ptnet_get_counter(if_t ifp, ift_counter cnt) 1018a9e644cdSLuigi Rizzo { 1019a9e644cdSLuigi Rizzo struct ptnet_softc *sc = if_getsoftc(ifp); 1020a9e644cdSLuigi Rizzo struct ptnet_queue_stats stats[2]; 1021a9e644cdSLuigi Rizzo int i; 1022a9e644cdSLuigi Rizzo 1023a9e644cdSLuigi Rizzo /* Accumulate statistics over the queues. */ 1024a9e644cdSLuigi Rizzo memset(stats, 0, sizeof(stats)); 1025a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 1026a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 1027a9e644cdSLuigi Rizzo int idx = (i < sc->num_tx_rings) ? 0 : 1; 1028a9e644cdSLuigi Rizzo 1029a9e644cdSLuigi Rizzo stats[idx].packets += pq->stats.packets; 1030a9e644cdSLuigi Rizzo stats[idx].bytes += pq->stats.bytes; 1031a9e644cdSLuigi Rizzo stats[idx].errors += pq->stats.errors; 1032a9e644cdSLuigi Rizzo stats[idx].iqdrops += pq->stats.iqdrops; 1033a9e644cdSLuigi Rizzo stats[idx].mcasts += pq->stats.mcasts; 1034a9e644cdSLuigi Rizzo } 1035a9e644cdSLuigi Rizzo 1036a9e644cdSLuigi Rizzo switch (cnt) { 1037a9e644cdSLuigi Rizzo case IFCOUNTER_IPACKETS: 1038a9e644cdSLuigi Rizzo return (stats[1].packets); 1039a9e644cdSLuigi Rizzo case IFCOUNTER_IQDROPS: 1040a9e644cdSLuigi Rizzo return (stats[1].iqdrops); 1041a9e644cdSLuigi Rizzo case IFCOUNTER_IERRORS: 1042a9e644cdSLuigi Rizzo return (stats[1].errors); 1043a9e644cdSLuigi Rizzo case IFCOUNTER_OPACKETS: 1044a9e644cdSLuigi Rizzo return (stats[0].packets); 1045a9e644cdSLuigi Rizzo case IFCOUNTER_OBYTES: 1046a9e644cdSLuigi Rizzo return (stats[0].bytes); 1047a9e644cdSLuigi Rizzo case IFCOUNTER_OMCASTS: 1048a9e644cdSLuigi Rizzo return (stats[0].mcasts); 1049a9e644cdSLuigi Rizzo default: 1050a9e644cdSLuigi Rizzo return (if_get_counter_default(ifp, cnt)); 1051a9e644cdSLuigi Rizzo } 1052a9e644cdSLuigi Rizzo } 1053a9e644cdSLuigi Rizzo #endif 1054a9e644cdSLuigi Rizzo 1055a9e644cdSLuigi Rizzo 1056a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS 1057a9e644cdSLuigi Rizzo /* Called under core lock. */ 1058a9e644cdSLuigi Rizzo static void 1059a9e644cdSLuigi Rizzo ptnet_tick(void *opaque) 1060a9e644cdSLuigi Rizzo { 1061a9e644cdSLuigi Rizzo struct ptnet_softc *sc = opaque; 1062a9e644cdSLuigi Rizzo int i; 1063a9e644cdSLuigi Rizzo 1064a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 1065a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 1066a9e644cdSLuigi Rizzo struct ptnet_queue_stats cur = pq->stats; 1067a9e644cdSLuigi Rizzo struct timeval now; 1068a9e644cdSLuigi Rizzo unsigned int delta; 1069a9e644cdSLuigi Rizzo 1070a9e644cdSLuigi Rizzo microtime(&now); 1071a9e644cdSLuigi Rizzo delta = now.tv_usec - sc->last_ts.tv_usec + 1072a9e644cdSLuigi Rizzo (now.tv_sec - sc->last_ts.tv_sec) * 1000000; 1073a9e644cdSLuigi Rizzo delta /= 1000; /* in milliseconds */ 1074a9e644cdSLuigi Rizzo 1075a9e644cdSLuigi Rizzo if (delta == 0) 1076a9e644cdSLuigi Rizzo continue; 1077a9e644cdSLuigi Rizzo 1078a9e644cdSLuigi Rizzo device_printf(sc->dev, "#%d[%u ms]:pkts %lu, kicks %lu, " 1079a9e644cdSLuigi Rizzo "intr %lu\n", i, delta, 1080a9e644cdSLuigi Rizzo (cur.packets - pq->last_stats.packets), 1081a9e644cdSLuigi Rizzo (cur.kicks - pq->last_stats.kicks), 1082a9e644cdSLuigi Rizzo (cur.intrs - pq->last_stats.intrs)); 1083a9e644cdSLuigi Rizzo pq->last_stats = cur; 1084a9e644cdSLuigi Rizzo } 1085a9e644cdSLuigi Rizzo microtime(&sc->last_ts); 1086a9e644cdSLuigi Rizzo callout_schedule(&sc->tick, hz); 1087a9e644cdSLuigi Rizzo } 1088a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */ 1089a9e644cdSLuigi Rizzo 1090a9e644cdSLuigi Rizzo static void 1091a9e644cdSLuigi Rizzo ptnet_media_status(if_t ifp, struct ifmediareq *ifmr) 1092a9e644cdSLuigi Rizzo { 1093a9e644cdSLuigi Rizzo /* We are always active, as the backend netmap port is 1094a9e644cdSLuigi Rizzo * always open in netmap mode. */ 1095a9e644cdSLuigi Rizzo ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE; 1096a9e644cdSLuigi Rizzo ifmr->ifm_active = IFM_ETHER | IFM_10G_T | IFM_FDX; 1097a9e644cdSLuigi Rizzo } 1098a9e644cdSLuigi Rizzo 1099a9e644cdSLuigi Rizzo static uint32_t 1100b6e66be2SVincenzo Maffione ptnet_nm_ptctl(struct ptnet_softc *sc, uint32_t cmd) 1101a9e644cdSLuigi Rizzo { 1102844a6f0cSLuigi Rizzo /* 1103844a6f0cSLuigi Rizzo * Write a command and read back error status, 1104844a6f0cSLuigi Rizzo * with zero meaning success. 1105844a6f0cSLuigi Rizzo */ 1106a9e644cdSLuigi Rizzo bus_write_4(sc->iomem, PTNET_IO_PTCTL, cmd); 1107844a6f0cSLuigi Rizzo return bus_read_4(sc->iomem, PTNET_IO_PTCTL); 1108a9e644cdSLuigi Rizzo } 1109a9e644cdSLuigi Rizzo 1110a9e644cdSLuigi Rizzo static int 11112ff91c17SVincenzo Maffione ptnet_nm_config(struct netmap_adapter *na, struct nm_config_info *info) 1112a9e644cdSLuigi Rizzo { 1113a9e644cdSLuigi Rizzo struct ptnet_softc *sc = if_getsoftc(na->ifp); 1114a9e644cdSLuigi Rizzo 11152ff91c17SVincenzo Maffione info->num_tx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_RINGS); 11162ff91c17SVincenzo Maffione info->num_rx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_RINGS); 11172ff91c17SVincenzo Maffione info->num_tx_descs = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_SLOTS); 11182ff91c17SVincenzo Maffione info->num_rx_descs = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_SLOTS); 11192ff91c17SVincenzo Maffione info->rx_buf_maxsize = NETMAP_BUF_SIZE(na); 1120a9e644cdSLuigi Rizzo 11212ff91c17SVincenzo Maffione device_printf(sc->dev, "txr %u, rxr %u, txd %u, rxd %u, rxbufsz %u\n", 11222ff91c17SVincenzo Maffione info->num_tx_rings, info->num_rx_rings, 11232ff91c17SVincenzo Maffione info->num_tx_descs, info->num_rx_descs, 11242ff91c17SVincenzo Maffione info->rx_buf_maxsize); 1125a9e644cdSLuigi Rizzo 1126a9e644cdSLuigi Rizzo return 0; 1127a9e644cdSLuigi Rizzo } 1128a9e644cdSLuigi Rizzo 1129a9e644cdSLuigi Rizzo static void 1130a9e644cdSLuigi Rizzo ptnet_sync_from_csb(struct ptnet_softc *sc, struct netmap_adapter *na) 1131a9e644cdSLuigi Rizzo { 1132a9e644cdSLuigi Rizzo int i; 1133a9e644cdSLuigi Rizzo 1134a9e644cdSLuigi Rizzo /* Sync krings from the host, reading from 1135a9e644cdSLuigi Rizzo * CSB. */ 1136a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 1137b6e66be2SVincenzo Maffione struct nm_csb_atok *atok = sc->queues[i].atok; 1138b6e66be2SVincenzo Maffione struct nm_csb_ktoa *ktoa = sc->queues[i].ktoa; 1139a9e644cdSLuigi Rizzo struct netmap_kring *kring; 1140a9e644cdSLuigi Rizzo 1141a9e644cdSLuigi Rizzo if (i < na->num_tx_rings) { 11422ff91c17SVincenzo Maffione kring = na->tx_rings[i]; 1143a9e644cdSLuigi Rizzo } else { 11442ff91c17SVincenzo Maffione kring = na->rx_rings[i - na->num_tx_rings]; 1145a9e644cdSLuigi Rizzo } 1146b6e66be2SVincenzo Maffione kring->rhead = kring->ring->head = atok->head; 1147b6e66be2SVincenzo Maffione kring->rcur = kring->ring->cur = atok->cur; 1148b6e66be2SVincenzo Maffione kring->nr_hwcur = ktoa->hwcur; 1149a9e644cdSLuigi Rizzo kring->nr_hwtail = kring->rtail = 1150b6e66be2SVincenzo Maffione kring->ring->tail = ktoa->hwtail; 1151a9e644cdSLuigi Rizzo 115275f4f3edSVincenzo Maffione nm_prdis("%d,%d: csb {hc %u h %u c %u ht %u}", t, i, 1153b6e66be2SVincenzo Maffione ktoa->hwcur, atok->head, atok->cur, 1154b6e66be2SVincenzo Maffione ktoa->hwtail); 115575f4f3edSVincenzo Maffione nm_prdis("%d,%d: kring {hc %u rh %u rc %u h %u c %u ht %u rt %u t %u}", 1156a9e644cdSLuigi Rizzo t, i, kring->nr_hwcur, kring->rhead, kring->rcur, 1157a9e644cdSLuigi Rizzo kring->ring->head, kring->ring->cur, kring->nr_hwtail, 1158a9e644cdSLuigi Rizzo kring->rtail, kring->ring->tail); 1159a9e644cdSLuigi Rizzo } 1160a9e644cdSLuigi Rizzo } 1161a9e644cdSLuigi Rizzo 1162a9e644cdSLuigi Rizzo static void 1163a9e644cdSLuigi Rizzo ptnet_update_vnet_hdr(struct ptnet_softc *sc) 1164a9e644cdSLuigi Rizzo { 1165a2a74091SLuigi Rizzo unsigned int wanted_hdr_len = ptnet_vnet_hdr ? PTNET_HDR_SIZE : 0; 1166a2a74091SLuigi Rizzo 1167a2a74091SLuigi Rizzo bus_write_4(sc->iomem, PTNET_IO_VNET_HDR_LEN, wanted_hdr_len); 1168a2a74091SLuigi Rizzo sc->vnet_hdr_len = bus_read_4(sc->iomem, PTNET_IO_VNET_HDR_LEN); 1169a9e644cdSLuigi Rizzo sc->ptna->hwup.up.virt_hdr_len = sc->vnet_hdr_len; 1170a9e644cdSLuigi Rizzo } 1171a9e644cdSLuigi Rizzo 1172a9e644cdSLuigi Rizzo static int 1173a9e644cdSLuigi Rizzo ptnet_nm_register(struct netmap_adapter *na, int onoff) 1174a9e644cdSLuigi Rizzo { 1175a9e644cdSLuigi Rizzo /* device-specific */ 1176a9e644cdSLuigi Rizzo if_t ifp = na->ifp; 1177a9e644cdSLuigi Rizzo struct ptnet_softc *sc = if_getsoftc(ifp); 1178a9e644cdSLuigi Rizzo int native = (na == &sc->ptna->hwup.up); 1179a9e644cdSLuigi Rizzo struct ptnet_queue *pq; 1180a9e644cdSLuigi Rizzo int ret = 0; 1181a9e644cdSLuigi Rizzo int i; 1182a9e644cdSLuigi Rizzo 1183a9e644cdSLuigi Rizzo if (!onoff) { 1184b6e66be2SVincenzo Maffione sc->ptna->backend_users--; 1185a9e644cdSLuigi Rizzo } 1186a9e644cdSLuigi Rizzo 1187a9e644cdSLuigi Rizzo /* If this is the last netmap client, guest interrupt enable flags may 1188a9e644cdSLuigi Rizzo * be in arbitrary state. Since these flags are going to be used also 1189a9e644cdSLuigi Rizzo * by the netdevice driver, we have to make sure to start with 1190a9e644cdSLuigi Rizzo * notifications enabled. Also, schedule NAPI to flush pending packets 1191a9e644cdSLuigi Rizzo * in the RX rings, since we will not receive further interrupts 1192a9e644cdSLuigi Rizzo * until these will be processed. */ 1193a9e644cdSLuigi Rizzo if (native && !onoff && na->active_fds == 0) { 119475f4f3edSVincenzo Maffione nm_prinf("Exit netmap mode, re-enable interrupts"); 1195a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 1196a9e644cdSLuigi Rizzo pq = sc->queues + i; 1197b6e66be2SVincenzo Maffione pq->atok->appl_need_kick = 1; 1198a9e644cdSLuigi Rizzo } 1199a9e644cdSLuigi Rizzo } 1200a9e644cdSLuigi Rizzo 1201a9e644cdSLuigi Rizzo if (onoff) { 1202b6e66be2SVincenzo Maffione if (sc->ptna->backend_users == 0) { 1203a9e644cdSLuigi Rizzo /* Initialize notification enable fields in the CSB. */ 1204a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 1205a9e644cdSLuigi Rizzo pq = sc->queues + i; 1206b6e66be2SVincenzo Maffione pq->ktoa->kern_need_kick = 1; 1207b6e66be2SVincenzo Maffione pq->atok->appl_need_kick = 1208a9e644cdSLuigi Rizzo (!(ifp->if_capenable & IFCAP_POLLING) 1209a9e644cdSLuigi Rizzo && i >= sc->num_tx_rings); 1210a9e644cdSLuigi Rizzo } 1211a9e644cdSLuigi Rizzo 1212a9e644cdSLuigi Rizzo /* Set the virtio-net header length. */ 1213a9e644cdSLuigi Rizzo ptnet_update_vnet_hdr(sc); 1214a9e644cdSLuigi Rizzo 1215a9e644cdSLuigi Rizzo /* Make sure the host adapter passed through is ready 1216a9e644cdSLuigi Rizzo * for txsync/rxsync. */ 1217b6e66be2SVincenzo Maffione ret = ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_CREATE); 1218a9e644cdSLuigi Rizzo if (ret) { 1219a9e644cdSLuigi Rizzo return ret; 1220a9e644cdSLuigi Rizzo } 1221a9e644cdSLuigi Rizzo 1222b6e66be2SVincenzo Maffione /* Align the guest krings and rings to the state stored 1223b6e66be2SVincenzo Maffione * in the CSB. */ 1224a9e644cdSLuigi Rizzo ptnet_sync_from_csb(sc, na); 1225a9e644cdSLuigi Rizzo } 1226a9e644cdSLuigi Rizzo 1227a9e644cdSLuigi Rizzo /* If not native, don't call nm_set_native_flags, since we don't want 1228a9e644cdSLuigi Rizzo * to replace if_transmit method, nor set NAF_NETMAP_ON */ 1229a9e644cdSLuigi Rizzo if (native) { 123075f4f3edSVincenzo Maffione netmap_krings_mode_commit(na, onoff); 1231a9e644cdSLuigi Rizzo nm_set_native_flags(na); 1232a9e644cdSLuigi Rizzo } 1233a9e644cdSLuigi Rizzo 1234a9e644cdSLuigi Rizzo } else { 1235a9e644cdSLuigi Rizzo if (native) { 1236a9e644cdSLuigi Rizzo nm_clear_native_flags(na); 123775f4f3edSVincenzo Maffione netmap_krings_mode_commit(na, onoff); 1238a9e644cdSLuigi Rizzo } 1239a9e644cdSLuigi Rizzo 1240b6e66be2SVincenzo Maffione if (sc->ptna->backend_users == 0) { 1241b6e66be2SVincenzo Maffione ret = ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_DELETE); 1242a9e644cdSLuigi Rizzo } 1243a9e644cdSLuigi Rizzo } 1244a9e644cdSLuigi Rizzo 1245a9e644cdSLuigi Rizzo if (onoff) { 1246b6e66be2SVincenzo Maffione sc->ptna->backend_users++; 1247a9e644cdSLuigi Rizzo } 1248a9e644cdSLuigi Rizzo 1249a9e644cdSLuigi Rizzo return ret; 1250a9e644cdSLuigi Rizzo } 1251a9e644cdSLuigi Rizzo 1252a9e644cdSLuigi Rizzo static int 1253a9e644cdSLuigi Rizzo ptnet_nm_txsync(struct netmap_kring *kring, int flags) 1254a9e644cdSLuigi Rizzo { 1255a9e644cdSLuigi Rizzo struct ptnet_softc *sc = if_getsoftc(kring->na->ifp); 1256a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + kring->ring_id; 1257a9e644cdSLuigi Rizzo bool notify; 1258a9e644cdSLuigi Rizzo 1259b6e66be2SVincenzo Maffione notify = netmap_pt_guest_txsync(pq->atok, pq->ktoa, kring, flags); 1260a9e644cdSLuigi Rizzo if (notify) { 1261a9e644cdSLuigi Rizzo ptnet_kick(pq); 1262a9e644cdSLuigi Rizzo } 1263a9e644cdSLuigi Rizzo 1264a9e644cdSLuigi Rizzo return 0; 1265a9e644cdSLuigi Rizzo } 1266a9e644cdSLuigi Rizzo 1267a9e644cdSLuigi Rizzo static int 1268a9e644cdSLuigi Rizzo ptnet_nm_rxsync(struct netmap_kring *kring, int flags) 1269a9e644cdSLuigi Rizzo { 1270a9e644cdSLuigi Rizzo struct ptnet_softc *sc = if_getsoftc(kring->na->ifp); 1271a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->rxqueues + kring->ring_id; 1272a9e644cdSLuigi Rizzo bool notify; 1273a9e644cdSLuigi Rizzo 1274b6e66be2SVincenzo Maffione notify = netmap_pt_guest_rxsync(pq->atok, pq->ktoa, kring, flags); 1275a9e644cdSLuigi Rizzo if (notify) { 1276a9e644cdSLuigi Rizzo ptnet_kick(pq); 1277a9e644cdSLuigi Rizzo } 1278a9e644cdSLuigi Rizzo 1279a9e644cdSLuigi Rizzo return 0; 1280a9e644cdSLuigi Rizzo } 1281a9e644cdSLuigi Rizzo 1282a9e644cdSLuigi Rizzo static void 12834f80b14cSVincenzo Maffione ptnet_nm_intr(struct netmap_adapter *na, int onoff) 12844f80b14cSVincenzo Maffione { 12854f80b14cSVincenzo Maffione struct ptnet_softc *sc = if_getsoftc(na->ifp); 12864f80b14cSVincenzo Maffione int i; 12874f80b14cSVincenzo Maffione 12884f80b14cSVincenzo Maffione for (i = 0; i < sc->num_rings; i++) { 12894f80b14cSVincenzo Maffione struct ptnet_queue *pq = sc->queues + i; 1290b6e66be2SVincenzo Maffione pq->atok->appl_need_kick = onoff; 12914f80b14cSVincenzo Maffione } 12924f80b14cSVincenzo Maffione } 12934f80b14cSVincenzo Maffione 12944f80b14cSVincenzo Maffione static void 1295a9e644cdSLuigi Rizzo ptnet_tx_intr(void *opaque) 1296a9e644cdSLuigi Rizzo { 1297a9e644cdSLuigi Rizzo struct ptnet_queue *pq = opaque; 1298a9e644cdSLuigi Rizzo struct ptnet_softc *sc = pq->sc; 1299a9e644cdSLuigi Rizzo 1300a9e644cdSLuigi Rizzo DBG(device_printf(sc->dev, "Tx interrupt #%d\n", pq->kring_id)); 1301a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS 1302a9e644cdSLuigi Rizzo pq->stats.intrs ++; 1303a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */ 1304a9e644cdSLuigi Rizzo 1305a9e644cdSLuigi Rizzo if (netmap_tx_irq(sc->ifp, pq->kring_id) != NM_IRQ_PASS) { 1306a9e644cdSLuigi Rizzo return; 1307a9e644cdSLuigi Rizzo } 1308a9e644cdSLuigi Rizzo 1309a9e644cdSLuigi Rizzo /* Schedule the tasqueue to flush process transmissions requests. 1310a9e644cdSLuigi Rizzo * However, vtnet, if_em and if_igb just call ptnet_transmit() here, 1311a9e644cdSLuigi Rizzo * at least when using MSI-X interrupts. The if_em driver, instead 1312a9e644cdSLuigi Rizzo * schedule taskqueue when using legacy interrupts. */ 1313a9e644cdSLuigi Rizzo taskqueue_enqueue(pq->taskq, &pq->task); 1314a9e644cdSLuigi Rizzo } 1315a9e644cdSLuigi Rizzo 1316a9e644cdSLuigi Rizzo static void 1317a9e644cdSLuigi Rizzo ptnet_rx_intr(void *opaque) 1318a9e644cdSLuigi Rizzo { 1319a9e644cdSLuigi Rizzo struct ptnet_queue *pq = opaque; 1320a9e644cdSLuigi Rizzo struct ptnet_softc *sc = pq->sc; 1321a9e644cdSLuigi Rizzo unsigned int unused; 1322a9e644cdSLuigi Rizzo 1323a9e644cdSLuigi Rizzo DBG(device_printf(sc->dev, "Rx interrupt #%d\n", pq->kring_id)); 1324a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS 1325a9e644cdSLuigi Rizzo pq->stats.intrs ++; 1326a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */ 1327a9e644cdSLuigi Rizzo 1328a9e644cdSLuigi Rizzo if (netmap_rx_irq(sc->ifp, pq->kring_id, &unused) != NM_IRQ_PASS) { 1329a9e644cdSLuigi Rizzo return; 1330a9e644cdSLuigi Rizzo } 1331a9e644cdSLuigi Rizzo 1332a9e644cdSLuigi Rizzo /* Like vtnet, if_igb and if_em drivers when using MSI-X interrupts, 1333a9e644cdSLuigi Rizzo * receive-side processing is executed directly in the interrupt 1334a9e644cdSLuigi Rizzo * service routine. Alternatively, we may schedule the taskqueue. */ 1335a9e644cdSLuigi Rizzo ptnet_rx_eof(pq, PTNET_RX_BUDGET, true); 1336a9e644cdSLuigi Rizzo } 1337a9e644cdSLuigi Rizzo 1338a9e644cdSLuigi Rizzo static void 1339a9e644cdSLuigi Rizzo ptnet_vlan_tag_remove(struct mbuf *m) 1340a9e644cdSLuigi Rizzo { 1341a9e644cdSLuigi Rizzo struct ether_vlan_header *evh; 1342a9e644cdSLuigi Rizzo 1343a9e644cdSLuigi Rizzo evh = mtod(m, struct ether_vlan_header *); 1344a9e644cdSLuigi Rizzo m->m_pkthdr.ether_vtag = ntohs(evh->evl_tag); 1345a9e644cdSLuigi Rizzo m->m_flags |= M_VLANTAG; 1346a9e644cdSLuigi Rizzo 1347a9e644cdSLuigi Rizzo /* Strip the 802.1Q header. */ 1348a9e644cdSLuigi Rizzo bcopy((char *) evh, (char *) evh + ETHER_VLAN_ENCAP_LEN, 1349a9e644cdSLuigi Rizzo ETHER_HDR_LEN - ETHER_TYPE_LEN); 1350a9e644cdSLuigi Rizzo m_adj(m, ETHER_VLAN_ENCAP_LEN); 1351a9e644cdSLuigi Rizzo } 1352a9e644cdSLuigi Rizzo 1353a9e644cdSLuigi Rizzo static void 1354a9e644cdSLuigi Rizzo ptnet_ring_update(struct ptnet_queue *pq, struct netmap_kring *kring, 1355a9e644cdSLuigi Rizzo unsigned int head, unsigned int sync_flags) 1356a9e644cdSLuigi Rizzo { 1357a9e644cdSLuigi Rizzo struct netmap_ring *ring = kring->ring; 1358b6e66be2SVincenzo Maffione struct nm_csb_atok *atok = pq->atok; 1359b6e66be2SVincenzo Maffione struct nm_csb_ktoa *ktoa = pq->ktoa; 1360a9e644cdSLuigi Rizzo 1361a9e644cdSLuigi Rizzo /* Some packets have been pushed to the netmap ring. We have 1362a9e644cdSLuigi Rizzo * to tell the host to process the new packets, updating cur 1363a9e644cdSLuigi Rizzo * and head in the CSB. */ 1364a9e644cdSLuigi Rizzo ring->head = ring->cur = head; 1365a9e644cdSLuigi Rizzo 1366a9e644cdSLuigi Rizzo /* Mimic nm_txsync_prologue/nm_rxsync_prologue. */ 1367a9e644cdSLuigi Rizzo kring->rcur = kring->rhead = head; 1368a9e644cdSLuigi Rizzo 1369f79ba6d7SVincenzo Maffione nm_sync_kloop_appl_write(atok, kring->rcur, kring->rhead); 1370a9e644cdSLuigi Rizzo 1371a9e644cdSLuigi Rizzo /* Kick the host if needed. */ 1372b6e66be2SVincenzo Maffione if (NM_ACCESS_ONCE(ktoa->kern_need_kick)) { 1373b6e66be2SVincenzo Maffione atok->sync_flags = sync_flags; 1374a9e644cdSLuigi Rizzo ptnet_kick(pq); 1375a9e644cdSLuigi Rizzo } 1376a9e644cdSLuigi Rizzo } 1377a9e644cdSLuigi Rizzo 1378a9e644cdSLuigi Rizzo #define PTNET_TX_NOSPACE(_h, _k, _min) \ 1379a9e644cdSLuigi Rizzo ((((_h) < (_k)->rtail) ? 0 : (_k)->nkr_num_slots) + \ 1380a9e644cdSLuigi Rizzo (_k)->rtail - (_h)) < (_min) 1381a9e644cdSLuigi Rizzo 1382a9e644cdSLuigi Rizzo /* This function may be called by the network stack, or by 1383a9e644cdSLuigi Rizzo * by the taskqueue thread. */ 1384a9e644cdSLuigi Rizzo static int 1385a9e644cdSLuigi Rizzo ptnet_drain_transmit_queue(struct ptnet_queue *pq, unsigned int budget, 1386a9e644cdSLuigi Rizzo bool may_resched) 1387a9e644cdSLuigi Rizzo { 1388a9e644cdSLuigi Rizzo struct ptnet_softc *sc = pq->sc; 1389a9e644cdSLuigi Rizzo bool have_vnet_hdr = sc->vnet_hdr_len; 1390a9e644cdSLuigi Rizzo struct netmap_adapter *na = &sc->ptna->dr.up; 1391a9e644cdSLuigi Rizzo if_t ifp = sc->ifp; 1392a9e644cdSLuigi Rizzo unsigned int batch_count = 0; 1393b6e66be2SVincenzo Maffione struct nm_csb_atok *atok; 1394b6e66be2SVincenzo Maffione struct nm_csb_ktoa *ktoa; 1395a9e644cdSLuigi Rizzo struct netmap_kring *kring; 1396a9e644cdSLuigi Rizzo struct netmap_ring *ring; 1397a9e644cdSLuigi Rizzo struct netmap_slot *slot; 1398a9e644cdSLuigi Rizzo unsigned int count = 0; 1399a9e644cdSLuigi Rizzo unsigned int minspace; 1400a9e644cdSLuigi Rizzo unsigned int head; 1401a9e644cdSLuigi Rizzo unsigned int lim; 1402a9e644cdSLuigi Rizzo struct mbuf *mhead; 1403a9e644cdSLuigi Rizzo struct mbuf *mf; 1404a9e644cdSLuigi Rizzo int nmbuf_bytes; 1405a9e644cdSLuigi Rizzo uint8_t *nmbuf; 1406a9e644cdSLuigi Rizzo 1407a9e644cdSLuigi Rizzo if (!PTNET_Q_TRYLOCK(pq)) { 1408a9e644cdSLuigi Rizzo /* We failed to acquire the lock, schedule the taskqueue. */ 140975f4f3edSVincenzo Maffione nm_prlim(1, "Deferring TX work"); 1410a9e644cdSLuigi Rizzo if (may_resched) { 1411a9e644cdSLuigi Rizzo taskqueue_enqueue(pq->taskq, &pq->task); 1412a9e644cdSLuigi Rizzo } 1413a9e644cdSLuigi Rizzo 1414a9e644cdSLuigi Rizzo return 0; 1415a9e644cdSLuigi Rizzo } 1416a9e644cdSLuigi Rizzo 1417a9e644cdSLuigi Rizzo if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) { 1418a9e644cdSLuigi Rizzo PTNET_Q_UNLOCK(pq); 141975f4f3edSVincenzo Maffione nm_prlim(1, "Interface is down"); 1420a9e644cdSLuigi Rizzo return ENETDOWN; 1421a9e644cdSLuigi Rizzo } 1422a9e644cdSLuigi Rizzo 1423b6e66be2SVincenzo Maffione atok = pq->atok; 1424b6e66be2SVincenzo Maffione ktoa = pq->ktoa; 14252ff91c17SVincenzo Maffione kring = na->tx_rings[pq->kring_id]; 1426a9e644cdSLuigi Rizzo ring = kring->ring; 1427a9e644cdSLuigi Rizzo lim = kring->nkr_num_slots - 1; 1428a9e644cdSLuigi Rizzo head = ring->head; 1429a9e644cdSLuigi Rizzo minspace = sc->min_tx_space; 1430a9e644cdSLuigi Rizzo 1431a9e644cdSLuigi Rizzo while (count < budget) { 1432a9e644cdSLuigi Rizzo if (PTNET_TX_NOSPACE(head, kring, minspace)) { 1433a9e644cdSLuigi Rizzo /* We ran out of slot, let's see if the host has 1434a9e644cdSLuigi Rizzo * freed up some, by reading hwcur and hwtail from 1435a9e644cdSLuigi Rizzo * the CSB. */ 1436b6e66be2SVincenzo Maffione ptnet_sync_tail(ktoa, kring); 1437a9e644cdSLuigi Rizzo 1438a9e644cdSLuigi Rizzo if (PTNET_TX_NOSPACE(head, kring, minspace)) { 1439a9e644cdSLuigi Rizzo /* Still no slots available. Reactivate the 1440a9e644cdSLuigi Rizzo * interrupts so that we can be notified 1441a9e644cdSLuigi Rizzo * when some free slots are made available by 1442a9e644cdSLuigi Rizzo * the host. */ 1443b6e66be2SVincenzo Maffione atok->appl_need_kick = 1; 1444a9e644cdSLuigi Rizzo 1445f79ba6d7SVincenzo Maffione /* Double check. We need a full barrier to 1446f79ba6d7SVincenzo Maffione * prevent the store to atok->appl_need_kick 1447f79ba6d7SVincenzo Maffione * to be reordered with the load from 1448f79ba6d7SVincenzo Maffione * ktoa->hwcur and ktoa->hwtail (store-load 1449f79ba6d7SVincenzo Maffione * barrier). */ 1450f79ba6d7SVincenzo Maffione nm_stld_barrier(); 1451b6e66be2SVincenzo Maffione ptnet_sync_tail(ktoa, kring); 1452a9e644cdSLuigi Rizzo if (likely(PTNET_TX_NOSPACE(head, kring, 1453a9e644cdSLuigi Rizzo minspace))) { 1454a9e644cdSLuigi Rizzo break; 1455a9e644cdSLuigi Rizzo } 1456a9e644cdSLuigi Rizzo 145775f4f3edSVincenzo Maffione nm_prlim(1, "Found more slots by doublecheck"); 1458a9e644cdSLuigi Rizzo /* More slots were freed before reactivating 1459a9e644cdSLuigi Rizzo * the interrupts. */ 1460b6e66be2SVincenzo Maffione atok->appl_need_kick = 0; 1461a9e644cdSLuigi Rizzo } 1462a9e644cdSLuigi Rizzo } 1463a9e644cdSLuigi Rizzo 1464a9e644cdSLuigi Rizzo mhead = drbr_peek(ifp, pq->bufring); 1465a9e644cdSLuigi Rizzo if (!mhead) { 1466a9e644cdSLuigi Rizzo break; 1467a9e644cdSLuigi Rizzo } 1468a9e644cdSLuigi Rizzo 1469a9e644cdSLuigi Rizzo /* Initialize transmission state variables. */ 1470a9e644cdSLuigi Rizzo slot = ring->slot + head; 1471a9e644cdSLuigi Rizzo nmbuf = NMB(na, slot); 1472a9e644cdSLuigi Rizzo nmbuf_bytes = 0; 1473a9e644cdSLuigi Rizzo 1474a9e644cdSLuigi Rizzo /* If needed, prepare the virtio-net header at the beginning 1475a9e644cdSLuigi Rizzo * of the first slot. */ 1476a9e644cdSLuigi Rizzo if (have_vnet_hdr) { 1477a9e644cdSLuigi Rizzo struct virtio_net_hdr *vh = 1478a9e644cdSLuigi Rizzo (struct virtio_net_hdr *)nmbuf; 1479a9e644cdSLuigi Rizzo 1480a9e644cdSLuigi Rizzo /* For performance, we could replace this memset() with 1481a9e644cdSLuigi Rizzo * two 8-bytes-wide writes. */ 1482a9e644cdSLuigi Rizzo memset(nmbuf, 0, PTNET_HDR_SIZE); 1483a9e644cdSLuigi Rizzo if (mhead->m_pkthdr.csum_flags & PTNET_ALL_OFFLOAD) { 1484*f8bc74e2SVincenzo Maffione mhead = virtio_net_tx_offload(ifp, mhead, false, 1485a9e644cdSLuigi Rizzo vh); 1486a9e644cdSLuigi Rizzo if (unlikely(!mhead)) { 1487a9e644cdSLuigi Rizzo /* Packet dropped because errors 1488a9e644cdSLuigi Rizzo * occurred while preparing the vnet 1489a9e644cdSLuigi Rizzo * header. Let's go ahead with the next 1490a9e644cdSLuigi Rizzo * packet. */ 1491a9e644cdSLuigi Rizzo pq->stats.errors ++; 1492a9e644cdSLuigi Rizzo drbr_advance(ifp, pq->bufring); 1493a9e644cdSLuigi Rizzo continue; 1494a9e644cdSLuigi Rizzo } 1495a9e644cdSLuigi Rizzo } 149675f4f3edSVincenzo Maffione nm_prdis(1, "%s: [csum_flags %lX] vnet hdr: flags %x " 1497a9e644cdSLuigi Rizzo "csum_start %u csum_ofs %u hdr_len = %u " 1498a9e644cdSLuigi Rizzo "gso_size %u gso_type %x", __func__, 1499a9e644cdSLuigi Rizzo mhead->m_pkthdr.csum_flags, vh->flags, 1500a9e644cdSLuigi Rizzo vh->csum_start, vh->csum_offset, vh->hdr_len, 1501a9e644cdSLuigi Rizzo vh->gso_size, vh->gso_type); 1502a9e644cdSLuigi Rizzo 1503a9e644cdSLuigi Rizzo nmbuf += PTNET_HDR_SIZE; 1504a9e644cdSLuigi Rizzo nmbuf_bytes += PTNET_HDR_SIZE; 1505a9e644cdSLuigi Rizzo } 1506a9e644cdSLuigi Rizzo 1507a9e644cdSLuigi Rizzo for (mf = mhead; mf; mf = mf->m_next) { 1508a9e644cdSLuigi Rizzo uint8_t *mdata = mf->m_data; 1509a9e644cdSLuigi Rizzo int mlen = mf->m_len; 1510a9e644cdSLuigi Rizzo 1511a9e644cdSLuigi Rizzo for (;;) { 1512a9e644cdSLuigi Rizzo int copy = NETMAP_BUF_SIZE(na) - nmbuf_bytes; 1513a9e644cdSLuigi Rizzo 1514a9e644cdSLuigi Rizzo if (mlen < copy) { 1515a9e644cdSLuigi Rizzo copy = mlen; 1516a9e644cdSLuigi Rizzo } 1517a9e644cdSLuigi Rizzo memcpy(nmbuf, mdata, copy); 1518a9e644cdSLuigi Rizzo 1519a9e644cdSLuigi Rizzo mdata += copy; 1520a9e644cdSLuigi Rizzo mlen -= copy; 1521a9e644cdSLuigi Rizzo nmbuf += copy; 1522a9e644cdSLuigi Rizzo nmbuf_bytes += copy; 1523a9e644cdSLuigi Rizzo 1524a9e644cdSLuigi Rizzo if (!mlen) { 1525a9e644cdSLuigi Rizzo break; 1526a9e644cdSLuigi Rizzo } 1527a9e644cdSLuigi Rizzo 1528a9e644cdSLuigi Rizzo slot->len = nmbuf_bytes; 1529a9e644cdSLuigi Rizzo slot->flags = NS_MOREFRAG; 1530a9e644cdSLuigi Rizzo 1531a9e644cdSLuigi Rizzo head = nm_next(head, lim); 1532a9e644cdSLuigi Rizzo KASSERT(head != ring->tail, 1533a9e644cdSLuigi Rizzo ("Unexpectedly run out of TX space")); 1534a9e644cdSLuigi Rizzo slot = ring->slot + head; 1535a9e644cdSLuigi Rizzo nmbuf = NMB(na, slot); 1536a9e644cdSLuigi Rizzo nmbuf_bytes = 0; 1537a9e644cdSLuigi Rizzo } 1538a9e644cdSLuigi Rizzo } 1539a9e644cdSLuigi Rizzo 1540a9e644cdSLuigi Rizzo /* Complete last slot and update head. */ 1541a9e644cdSLuigi Rizzo slot->len = nmbuf_bytes; 1542a9e644cdSLuigi Rizzo slot->flags = 0; 1543a9e644cdSLuigi Rizzo head = nm_next(head, lim); 1544a9e644cdSLuigi Rizzo 1545a9e644cdSLuigi Rizzo /* Consume the packet just processed. */ 1546a9e644cdSLuigi Rizzo drbr_advance(ifp, pq->bufring); 1547a9e644cdSLuigi Rizzo 1548a9e644cdSLuigi Rizzo /* Copy the packet to listeners. */ 1549a9e644cdSLuigi Rizzo ETHER_BPF_MTAP(ifp, mhead); 1550a9e644cdSLuigi Rizzo 1551a9e644cdSLuigi Rizzo pq->stats.packets ++; 1552a9e644cdSLuigi Rizzo pq->stats.bytes += mhead->m_pkthdr.len; 1553a9e644cdSLuigi Rizzo if (mhead->m_flags & M_MCAST) { 1554a9e644cdSLuigi Rizzo pq->stats.mcasts ++; 1555a9e644cdSLuigi Rizzo } 1556a9e644cdSLuigi Rizzo 1557a9e644cdSLuigi Rizzo m_freem(mhead); 1558a9e644cdSLuigi Rizzo 1559a9e644cdSLuigi Rizzo count ++; 1560a9e644cdSLuigi Rizzo if (++batch_count == PTNET_TX_BATCH) { 1561a9e644cdSLuigi Rizzo ptnet_ring_update(pq, kring, head, NAF_FORCE_RECLAIM); 1562a9e644cdSLuigi Rizzo batch_count = 0; 1563a9e644cdSLuigi Rizzo } 1564a9e644cdSLuigi Rizzo } 1565a9e644cdSLuigi Rizzo 1566a9e644cdSLuigi Rizzo if (batch_count) { 1567a9e644cdSLuigi Rizzo ptnet_ring_update(pq, kring, head, NAF_FORCE_RECLAIM); 1568a9e644cdSLuigi Rizzo } 1569a9e644cdSLuigi Rizzo 1570a9e644cdSLuigi Rizzo if (count >= budget && may_resched) { 157175f4f3edSVincenzo Maffione DBG(nm_prlim(1, "out of budget: resched, %d mbufs pending\n", 1572a9e644cdSLuigi Rizzo drbr_inuse(ifp, pq->bufring))); 1573a9e644cdSLuigi Rizzo taskqueue_enqueue(pq->taskq, &pq->task); 1574a9e644cdSLuigi Rizzo } 1575a9e644cdSLuigi Rizzo 1576a9e644cdSLuigi Rizzo PTNET_Q_UNLOCK(pq); 1577a9e644cdSLuigi Rizzo 1578a9e644cdSLuigi Rizzo return count; 1579a9e644cdSLuigi Rizzo } 1580a9e644cdSLuigi Rizzo 1581a9e644cdSLuigi Rizzo static int 1582a9e644cdSLuigi Rizzo ptnet_transmit(if_t ifp, struct mbuf *m) 1583a9e644cdSLuigi Rizzo { 1584a9e644cdSLuigi Rizzo struct ptnet_softc *sc = if_getsoftc(ifp); 1585a9e644cdSLuigi Rizzo struct ptnet_queue *pq; 1586a9e644cdSLuigi Rizzo unsigned int queue_idx; 1587a9e644cdSLuigi Rizzo int err; 1588a9e644cdSLuigi Rizzo 1589a9e644cdSLuigi Rizzo DBG(device_printf(sc->dev, "transmit %p\n", m)); 1590a9e644cdSLuigi Rizzo 1591a9e644cdSLuigi Rizzo /* Insert 802.1Q header if needed. */ 1592a9e644cdSLuigi Rizzo if (m->m_flags & M_VLANTAG) { 1593a9e644cdSLuigi Rizzo m = ether_vlanencap(m, m->m_pkthdr.ether_vtag); 1594a9e644cdSLuigi Rizzo if (m == NULL) { 1595a9e644cdSLuigi Rizzo return ENOBUFS; 1596a9e644cdSLuigi Rizzo } 1597a9e644cdSLuigi Rizzo m->m_flags &= ~M_VLANTAG; 1598a9e644cdSLuigi Rizzo } 1599a9e644cdSLuigi Rizzo 1600a9e644cdSLuigi Rizzo /* Get the flow-id if available. */ 1601a9e644cdSLuigi Rizzo queue_idx = (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) ? 1602a9e644cdSLuigi Rizzo m->m_pkthdr.flowid : curcpu; 1603a9e644cdSLuigi Rizzo 1604a9e644cdSLuigi Rizzo if (unlikely(queue_idx >= sc->num_tx_rings)) { 1605a9e644cdSLuigi Rizzo queue_idx %= sc->num_tx_rings; 1606a9e644cdSLuigi Rizzo } 1607a9e644cdSLuigi Rizzo 1608a9e644cdSLuigi Rizzo pq = sc->queues + queue_idx; 1609a9e644cdSLuigi Rizzo 1610a9e644cdSLuigi Rizzo err = drbr_enqueue(ifp, pq->bufring, m); 1611a9e644cdSLuigi Rizzo if (err) { 1612a9e644cdSLuigi Rizzo /* ENOBUFS when the bufring is full */ 161375f4f3edSVincenzo Maffione nm_prlim(1, "%s: drbr_enqueue() failed %d\n", 1614a9e644cdSLuigi Rizzo __func__, err); 1615a9e644cdSLuigi Rizzo pq->stats.errors ++; 1616a9e644cdSLuigi Rizzo return err; 1617a9e644cdSLuigi Rizzo } 1618a9e644cdSLuigi Rizzo 1619a9e644cdSLuigi Rizzo if (ifp->if_capenable & IFCAP_POLLING) { 1620a9e644cdSLuigi Rizzo /* If polling is on, the transmit queues will be 1621a9e644cdSLuigi Rizzo * drained by the poller. */ 1622a9e644cdSLuigi Rizzo return 0; 1623a9e644cdSLuigi Rizzo } 1624a9e644cdSLuigi Rizzo 1625a9e644cdSLuigi Rizzo err = ptnet_drain_transmit_queue(pq, PTNET_TX_BUDGET, true); 1626a9e644cdSLuigi Rizzo 1627a9e644cdSLuigi Rizzo return (err < 0) ? err : 0; 1628a9e644cdSLuigi Rizzo } 1629a9e644cdSLuigi Rizzo 1630a9e644cdSLuigi Rizzo static unsigned int 1631a9e644cdSLuigi Rizzo ptnet_rx_discard(struct netmap_kring *kring, unsigned int head) 1632a9e644cdSLuigi Rizzo { 1633a9e644cdSLuigi Rizzo struct netmap_ring *ring = kring->ring; 1634a9e644cdSLuigi Rizzo struct netmap_slot *slot = ring->slot + head; 1635a9e644cdSLuigi Rizzo 1636a9e644cdSLuigi Rizzo for (;;) { 1637a9e644cdSLuigi Rizzo head = nm_next(head, kring->nkr_num_slots - 1); 1638a9e644cdSLuigi Rizzo if (!(slot->flags & NS_MOREFRAG) || head == ring->tail) { 1639a9e644cdSLuigi Rizzo break; 1640a9e644cdSLuigi Rizzo } 1641a9e644cdSLuigi Rizzo slot = ring->slot + head; 1642a9e644cdSLuigi Rizzo } 1643a9e644cdSLuigi Rizzo 1644a9e644cdSLuigi Rizzo return head; 1645a9e644cdSLuigi Rizzo } 1646a9e644cdSLuigi Rizzo 1647a9e644cdSLuigi Rizzo static inline struct mbuf * 1648a9e644cdSLuigi Rizzo ptnet_rx_slot(struct mbuf *mtail, uint8_t *nmbuf, unsigned int nmbuf_len) 1649a9e644cdSLuigi Rizzo { 1650a9e644cdSLuigi Rizzo uint8_t *mdata = mtod(mtail, uint8_t *) + mtail->m_len; 1651a9e644cdSLuigi Rizzo 1652a9e644cdSLuigi Rizzo do { 1653a9e644cdSLuigi Rizzo unsigned int copy; 1654a9e644cdSLuigi Rizzo 1655a9e644cdSLuigi Rizzo if (mtail->m_len == MCLBYTES) { 1656a9e644cdSLuigi Rizzo struct mbuf *mf; 1657a9e644cdSLuigi Rizzo 1658a9e644cdSLuigi Rizzo mf = m_getcl(M_NOWAIT, MT_DATA, 0); 1659a9e644cdSLuigi Rizzo if (unlikely(!mf)) { 1660a9e644cdSLuigi Rizzo return NULL; 1661a9e644cdSLuigi Rizzo } 1662a9e644cdSLuigi Rizzo 1663a9e644cdSLuigi Rizzo mtail->m_next = mf; 1664a9e644cdSLuigi Rizzo mtail = mf; 1665a9e644cdSLuigi Rizzo mdata = mtod(mtail, uint8_t *); 1666a9e644cdSLuigi Rizzo mtail->m_len = 0; 1667a9e644cdSLuigi Rizzo } 1668a9e644cdSLuigi Rizzo 1669a9e644cdSLuigi Rizzo copy = MCLBYTES - mtail->m_len; 1670a9e644cdSLuigi Rizzo if (nmbuf_len < copy) { 1671a9e644cdSLuigi Rizzo copy = nmbuf_len; 1672a9e644cdSLuigi Rizzo } 1673a9e644cdSLuigi Rizzo 1674a9e644cdSLuigi Rizzo memcpy(mdata, nmbuf, copy); 1675a9e644cdSLuigi Rizzo 1676a9e644cdSLuigi Rizzo nmbuf += copy; 1677a9e644cdSLuigi Rizzo nmbuf_len -= copy; 1678a9e644cdSLuigi Rizzo mdata += copy; 1679a9e644cdSLuigi Rizzo mtail->m_len += copy; 1680a9e644cdSLuigi Rizzo } while (nmbuf_len); 1681a9e644cdSLuigi Rizzo 1682a9e644cdSLuigi Rizzo return mtail; 1683a9e644cdSLuigi Rizzo } 1684a9e644cdSLuigi Rizzo 1685a9e644cdSLuigi Rizzo static int 1686a9e644cdSLuigi Rizzo ptnet_rx_eof(struct ptnet_queue *pq, unsigned int budget, bool may_resched) 1687a9e644cdSLuigi Rizzo { 1688a9e644cdSLuigi Rizzo struct ptnet_softc *sc = pq->sc; 1689a9e644cdSLuigi Rizzo bool have_vnet_hdr = sc->vnet_hdr_len; 1690b6e66be2SVincenzo Maffione struct nm_csb_atok *atok = pq->atok; 1691b6e66be2SVincenzo Maffione struct nm_csb_ktoa *ktoa = pq->ktoa; 1692a9e644cdSLuigi Rizzo struct netmap_adapter *na = &sc->ptna->dr.up; 16932ff91c17SVincenzo Maffione struct netmap_kring *kring = na->rx_rings[pq->kring_id]; 1694a9e644cdSLuigi Rizzo struct netmap_ring *ring = kring->ring; 1695a9e644cdSLuigi Rizzo unsigned int const lim = kring->nkr_num_slots - 1; 1696a9e644cdSLuigi Rizzo unsigned int batch_count = 0; 1697a9e644cdSLuigi Rizzo if_t ifp = sc->ifp; 1698a9e644cdSLuigi Rizzo unsigned int count = 0; 169946023447SVincenzo Maffione uint32_t head; 1700a9e644cdSLuigi Rizzo 1701a9e644cdSLuigi Rizzo PTNET_Q_LOCK(pq); 1702a9e644cdSLuigi Rizzo 1703a9e644cdSLuigi Rizzo if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) { 1704a9e644cdSLuigi Rizzo goto unlock; 1705a9e644cdSLuigi Rizzo } 1706a9e644cdSLuigi Rizzo 1707a9e644cdSLuigi Rizzo kring->nr_kflags &= ~NKR_PENDINTR; 1708a9e644cdSLuigi Rizzo 170946023447SVincenzo Maffione head = ring->head; 1710a9e644cdSLuigi Rizzo while (count < budget) { 171146023447SVincenzo Maffione uint32_t prev_head = head; 1712a9e644cdSLuigi Rizzo struct mbuf *mhead, *mtail; 1713a9e644cdSLuigi Rizzo struct virtio_net_hdr *vh; 1714a9e644cdSLuigi Rizzo struct netmap_slot *slot; 1715a9e644cdSLuigi Rizzo unsigned int nmbuf_len; 1716a9e644cdSLuigi Rizzo uint8_t *nmbuf; 171746023447SVincenzo Maffione int deliver = 1; /* the mbuf to the network stack. */ 1718a9e644cdSLuigi Rizzo host_sync: 1719a9e644cdSLuigi Rizzo if (head == ring->tail) { 1720a9e644cdSLuigi Rizzo /* We ran out of slot, let's see if the host has 1721a9e644cdSLuigi Rizzo * added some, by reading hwcur and hwtail from 1722a9e644cdSLuigi Rizzo * the CSB. */ 1723b6e66be2SVincenzo Maffione ptnet_sync_tail(ktoa, kring); 1724a9e644cdSLuigi Rizzo 1725a9e644cdSLuigi Rizzo if (head == ring->tail) { 1726a9e644cdSLuigi Rizzo /* Still no slots available. Reactivate 1727a9e644cdSLuigi Rizzo * interrupts as they were disabled by the 1728a9e644cdSLuigi Rizzo * host thread right before issuing the 1729a9e644cdSLuigi Rizzo * last interrupt. */ 1730b6e66be2SVincenzo Maffione atok->appl_need_kick = 1; 1731a9e644cdSLuigi Rizzo 1732f79ba6d7SVincenzo Maffione /* Double check for more completed RX slots. 1733f79ba6d7SVincenzo Maffione * We need a full barrier to prevent the store 1734f79ba6d7SVincenzo Maffione * to atok->appl_need_kick to be reordered with 1735f79ba6d7SVincenzo Maffione * the load from ktoa->hwcur and ktoa->hwtail 1736f79ba6d7SVincenzo Maffione * (store-load barrier). */ 1737f79ba6d7SVincenzo Maffione nm_stld_barrier(); 1738b6e66be2SVincenzo Maffione ptnet_sync_tail(ktoa, kring); 1739a9e644cdSLuigi Rizzo if (likely(head == ring->tail)) { 1740a9e644cdSLuigi Rizzo break; 1741a9e644cdSLuigi Rizzo } 1742b6e66be2SVincenzo Maffione atok->appl_need_kick = 0; 1743a9e644cdSLuigi Rizzo } 1744a9e644cdSLuigi Rizzo } 1745a9e644cdSLuigi Rizzo 1746a9e644cdSLuigi Rizzo /* Initialize ring state variables, possibly grabbing the 1747a9e644cdSLuigi Rizzo * virtio-net header. */ 1748a9e644cdSLuigi Rizzo slot = ring->slot + head; 1749a9e644cdSLuigi Rizzo nmbuf = NMB(na, slot); 1750a9e644cdSLuigi Rizzo nmbuf_len = slot->len; 1751a9e644cdSLuigi Rizzo 1752a9e644cdSLuigi Rizzo vh = (struct virtio_net_hdr *)nmbuf; 1753a9e644cdSLuigi Rizzo if (have_vnet_hdr) { 1754a9e644cdSLuigi Rizzo if (unlikely(nmbuf_len < PTNET_HDR_SIZE)) { 1755a9e644cdSLuigi Rizzo /* There is no good reason why host should 1756a9e644cdSLuigi Rizzo * put the header in multiple netmap slots. 1757a9e644cdSLuigi Rizzo * If this is the case, discard. */ 175875f4f3edSVincenzo Maffione nm_prlim(1, "Fragmented vnet-hdr: dropping"); 1759a9e644cdSLuigi Rizzo head = ptnet_rx_discard(kring, head); 1760a9e644cdSLuigi Rizzo pq->stats.iqdrops ++; 176146023447SVincenzo Maffione deliver = 0; 1762a9e644cdSLuigi Rizzo goto skip; 1763a9e644cdSLuigi Rizzo } 176475f4f3edSVincenzo Maffione nm_prdis(1, "%s: vnet hdr: flags %x csum_start %u " 1765a9e644cdSLuigi Rizzo "csum_ofs %u hdr_len = %u gso_size %u " 1766a9e644cdSLuigi Rizzo "gso_type %x", __func__, vh->flags, 1767a9e644cdSLuigi Rizzo vh->csum_start, vh->csum_offset, vh->hdr_len, 1768a9e644cdSLuigi Rizzo vh->gso_size, vh->gso_type); 1769a9e644cdSLuigi Rizzo nmbuf += PTNET_HDR_SIZE; 1770a9e644cdSLuigi Rizzo nmbuf_len -= PTNET_HDR_SIZE; 1771a9e644cdSLuigi Rizzo } 1772a9e644cdSLuigi Rizzo 1773a9e644cdSLuigi Rizzo /* Allocate the head of a new mbuf chain. 1774a9e644cdSLuigi Rizzo * We use m_getcl() to allocate an mbuf with standard cluster 1775a9e644cdSLuigi Rizzo * size (MCLBYTES). In the future we could use m_getjcl() 1776a9e644cdSLuigi Rizzo * to choose different sizes. */ 1777a9e644cdSLuigi Rizzo mhead = mtail = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 1778a9e644cdSLuigi Rizzo if (unlikely(mhead == NULL)) { 1779a9e644cdSLuigi Rizzo device_printf(sc->dev, "%s: failed to allocate mbuf " 1780a9e644cdSLuigi Rizzo "head\n", __func__); 1781a9e644cdSLuigi Rizzo pq->stats.errors ++; 1782a9e644cdSLuigi Rizzo break; 1783a9e644cdSLuigi Rizzo } 1784a9e644cdSLuigi Rizzo 1785a9e644cdSLuigi Rizzo /* Initialize the mbuf state variables. */ 1786a9e644cdSLuigi Rizzo mhead->m_pkthdr.len = nmbuf_len; 1787a9e644cdSLuigi Rizzo mtail->m_len = 0; 1788a9e644cdSLuigi Rizzo 1789a9e644cdSLuigi Rizzo /* Scan all the netmap slots containing the current packet. */ 1790a9e644cdSLuigi Rizzo for (;;) { 1791a9e644cdSLuigi Rizzo DBG(device_printf(sc->dev, "%s: h %u t %u rcv frag " 1792a9e644cdSLuigi Rizzo "len %u, flags %u\n", __func__, 1793a9e644cdSLuigi Rizzo head, ring->tail, slot->len, 1794a9e644cdSLuigi Rizzo slot->flags)); 1795a9e644cdSLuigi Rizzo 1796a9e644cdSLuigi Rizzo mtail = ptnet_rx_slot(mtail, nmbuf, nmbuf_len); 1797a9e644cdSLuigi Rizzo if (unlikely(!mtail)) { 1798a9e644cdSLuigi Rizzo /* Ouch. We ran out of memory while processing 1799a9e644cdSLuigi Rizzo * a packet. We have to restore the previous 1800a9e644cdSLuigi Rizzo * head position, free the mbuf chain, and 1801a9e644cdSLuigi Rizzo * schedule the taskqueue to give the packet 1802a9e644cdSLuigi Rizzo * another chance. */ 1803a9e644cdSLuigi Rizzo device_printf(sc->dev, "%s: failed to allocate" 1804a9e644cdSLuigi Rizzo " mbuf frag, reset head %u --> %u\n", 1805a9e644cdSLuigi Rizzo __func__, head, prev_head); 1806a9e644cdSLuigi Rizzo head = prev_head; 1807a9e644cdSLuigi Rizzo m_freem(mhead); 1808a9e644cdSLuigi Rizzo pq->stats.errors ++; 1809a9e644cdSLuigi Rizzo if (may_resched) { 1810a9e644cdSLuigi Rizzo taskqueue_enqueue(pq->taskq, 1811a9e644cdSLuigi Rizzo &pq->task); 1812a9e644cdSLuigi Rizzo } 1813a9e644cdSLuigi Rizzo goto escape; 1814a9e644cdSLuigi Rizzo } 1815a9e644cdSLuigi Rizzo 1816a9e644cdSLuigi Rizzo /* We have to increment head irrespective of the 1817a9e644cdSLuigi Rizzo * NS_MOREFRAG being set or not. */ 1818a9e644cdSLuigi Rizzo head = nm_next(head, lim); 1819a9e644cdSLuigi Rizzo 1820a9e644cdSLuigi Rizzo if (!(slot->flags & NS_MOREFRAG)) { 1821a9e644cdSLuigi Rizzo break; 1822a9e644cdSLuigi Rizzo } 1823a9e644cdSLuigi Rizzo 1824a9e644cdSLuigi Rizzo if (unlikely(head == ring->tail)) { 1825a9e644cdSLuigi Rizzo /* The very last slot prepared by the host has 1826a9e644cdSLuigi Rizzo * the NS_MOREFRAG set. Drop it and continue 1827a9e644cdSLuigi Rizzo * the outer cycle (to do the double-check). */ 182875f4f3edSVincenzo Maffione nm_prlim(1, "Incomplete packet: dropping"); 1829a9e644cdSLuigi Rizzo m_freem(mhead); 1830a9e644cdSLuigi Rizzo pq->stats.iqdrops ++; 1831a9e644cdSLuigi Rizzo goto host_sync; 1832a9e644cdSLuigi Rizzo } 1833a9e644cdSLuigi Rizzo 1834a9e644cdSLuigi Rizzo slot = ring->slot + head; 1835a9e644cdSLuigi Rizzo nmbuf = NMB(na, slot); 1836a9e644cdSLuigi Rizzo nmbuf_len = slot->len; 1837a9e644cdSLuigi Rizzo mhead->m_pkthdr.len += nmbuf_len; 1838a9e644cdSLuigi Rizzo } 1839a9e644cdSLuigi Rizzo 1840a9e644cdSLuigi Rizzo mhead->m_pkthdr.rcvif = ifp; 1841a9e644cdSLuigi Rizzo mhead->m_pkthdr.csum_flags = 0; 1842a9e644cdSLuigi Rizzo 1843a9e644cdSLuigi Rizzo /* Store the queue idx in the packet header. */ 1844a9e644cdSLuigi Rizzo mhead->m_pkthdr.flowid = pq->kring_id; 1845a9e644cdSLuigi Rizzo M_HASHTYPE_SET(mhead, M_HASHTYPE_OPAQUE); 1846a9e644cdSLuigi Rizzo 1847a9e644cdSLuigi Rizzo if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { 1848a9e644cdSLuigi Rizzo struct ether_header *eh; 1849a9e644cdSLuigi Rizzo 1850a9e644cdSLuigi Rizzo eh = mtod(mhead, struct ether_header *); 1851a9e644cdSLuigi Rizzo if (eh->ether_type == htons(ETHERTYPE_VLAN)) { 1852a9e644cdSLuigi Rizzo ptnet_vlan_tag_remove(mhead); 1853a9e644cdSLuigi Rizzo /* 1854a9e644cdSLuigi Rizzo * With the 802.1Q header removed, update the 1855a9e644cdSLuigi Rizzo * checksum starting location accordingly. 1856a9e644cdSLuigi Rizzo */ 1857a9e644cdSLuigi Rizzo if (vh->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) 1858a9e644cdSLuigi Rizzo vh->csum_start -= ETHER_VLAN_ENCAP_LEN; 1859a9e644cdSLuigi Rizzo } 1860a9e644cdSLuigi Rizzo } 1861a9e644cdSLuigi Rizzo 1862*f8bc74e2SVincenzo Maffione if (unlikely(have_vnet_hdr && virtio_net_rx_csum(mhead, vh))) { 1863a9e644cdSLuigi Rizzo m_freem(mhead); 186475f4f3edSVincenzo Maffione nm_prlim(1, "Csum offload error: dropping"); 1865a9e644cdSLuigi Rizzo pq->stats.iqdrops ++; 186646023447SVincenzo Maffione deliver = 0; 1867a9e644cdSLuigi Rizzo } 1868a9e644cdSLuigi Rizzo 186946023447SVincenzo Maffione skip: 187046023447SVincenzo Maffione count ++; 187146023447SVincenzo Maffione if (++batch_count >= PTNET_RX_BATCH) { 187246023447SVincenzo Maffione /* Some packets have been (or will be) pushed to the network 187346023447SVincenzo Maffione * stack. We need to update the CSB to tell the host about 187446023447SVincenzo Maffione * the new ring->cur and ring->head (RX buffer refill). */ 187546023447SVincenzo Maffione ptnet_ring_update(pq, kring, head, NAF_FORCE_READ); 187646023447SVincenzo Maffione batch_count = 0; 187746023447SVincenzo Maffione } 187846023447SVincenzo Maffione 187946023447SVincenzo Maffione if (likely(deliver)) { 1880a9e644cdSLuigi Rizzo pq->stats.packets ++; 1881a9e644cdSLuigi Rizzo pq->stats.bytes += mhead->m_pkthdr.len; 1882a9e644cdSLuigi Rizzo 1883a9e644cdSLuigi Rizzo PTNET_Q_UNLOCK(pq); 1884a9e644cdSLuigi Rizzo (*ifp->if_input)(ifp, mhead); 1885a9e644cdSLuigi Rizzo PTNET_Q_LOCK(pq); 188646023447SVincenzo Maffione /* The ring->head index (and related indices) are 188746023447SVincenzo Maffione * updated under pq lock by ptnet_ring_update(). 188846023447SVincenzo Maffione * Since we dropped the lock to call if_input(), we 188946023447SVincenzo Maffione * must reload ring->head and restart processing the 189046023447SVincenzo Maffione * ring from there. */ 189146023447SVincenzo Maffione head = ring->head; 1892a9e644cdSLuigi Rizzo 1893a9e644cdSLuigi Rizzo if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) { 1894a9e644cdSLuigi Rizzo /* The interface has gone down while we didn't 1895a9e644cdSLuigi Rizzo * have the lock. Stop any processing and exit. */ 1896a9e644cdSLuigi Rizzo goto unlock; 1897a9e644cdSLuigi Rizzo } 1898a9e644cdSLuigi Rizzo } 1899a9e644cdSLuigi Rizzo } 1900a9e644cdSLuigi Rizzo escape: 1901a9e644cdSLuigi Rizzo if (batch_count) { 1902a9e644cdSLuigi Rizzo ptnet_ring_update(pq, kring, head, NAF_FORCE_READ); 1903a9e644cdSLuigi Rizzo 1904a9e644cdSLuigi Rizzo } 1905a9e644cdSLuigi Rizzo 1906a9e644cdSLuigi Rizzo if (count >= budget && may_resched) { 1907a9e644cdSLuigi Rizzo /* If we ran out of budget or the double-check found new 1908a9e644cdSLuigi Rizzo * slots to process, schedule the taskqueue. */ 190975f4f3edSVincenzo Maffione DBG(nm_prlim(1, "out of budget: resched h %u t %u\n", 1910a9e644cdSLuigi Rizzo head, ring->tail)); 1911a9e644cdSLuigi Rizzo taskqueue_enqueue(pq->taskq, &pq->task); 1912a9e644cdSLuigi Rizzo } 1913a9e644cdSLuigi Rizzo unlock: 1914a9e644cdSLuigi Rizzo PTNET_Q_UNLOCK(pq); 1915a9e644cdSLuigi Rizzo 1916a9e644cdSLuigi Rizzo return count; 1917a9e644cdSLuigi Rizzo } 1918a9e644cdSLuigi Rizzo 1919a9e644cdSLuigi Rizzo static void 1920a9e644cdSLuigi Rizzo ptnet_rx_task(void *context, int pending) 1921a9e644cdSLuigi Rizzo { 1922a9e644cdSLuigi Rizzo struct ptnet_queue *pq = context; 1923a9e644cdSLuigi Rizzo 192475f4f3edSVincenzo Maffione DBG(nm_prlim(1, "%s: pq #%u\n", __func__, pq->kring_id)); 1925a9e644cdSLuigi Rizzo ptnet_rx_eof(pq, PTNET_RX_BUDGET, true); 1926a9e644cdSLuigi Rizzo } 1927a9e644cdSLuigi Rizzo 1928a9e644cdSLuigi Rizzo static void 1929a9e644cdSLuigi Rizzo ptnet_tx_task(void *context, int pending) 1930a9e644cdSLuigi Rizzo { 1931a9e644cdSLuigi Rizzo struct ptnet_queue *pq = context; 1932a9e644cdSLuigi Rizzo 193375f4f3edSVincenzo Maffione DBG(nm_prlim(1, "%s: pq #%u\n", __func__, pq->kring_id)); 1934a9e644cdSLuigi Rizzo ptnet_drain_transmit_queue(pq, PTNET_TX_BUDGET, true); 1935a9e644cdSLuigi Rizzo } 1936a9e644cdSLuigi Rizzo 1937a9e644cdSLuigi Rizzo #ifdef DEVICE_POLLING 1938a9e644cdSLuigi Rizzo /* We don't need to handle differently POLL_AND_CHECK_STATUS and 1939a9e644cdSLuigi Rizzo * POLL_ONLY, since we don't have an Interrupt Status Register. */ 1940a9e644cdSLuigi Rizzo static int 1941a9e644cdSLuigi Rizzo ptnet_poll(if_t ifp, enum poll_cmd cmd, int budget) 1942a9e644cdSLuigi Rizzo { 1943a9e644cdSLuigi Rizzo struct ptnet_softc *sc = if_getsoftc(ifp); 1944a9e644cdSLuigi Rizzo unsigned int queue_budget; 1945a9e644cdSLuigi Rizzo unsigned int count = 0; 1946a9e644cdSLuigi Rizzo bool borrow = false; 1947a9e644cdSLuigi Rizzo int i; 1948a9e644cdSLuigi Rizzo 1949a9e644cdSLuigi Rizzo KASSERT(sc->num_rings > 0, ("Found no queues in while polling ptnet")); 1950a9e644cdSLuigi Rizzo queue_budget = MAX(budget / sc->num_rings, 1); 195175f4f3edSVincenzo Maffione nm_prlim(1, "Per-queue budget is %d", queue_budget); 1952a9e644cdSLuigi Rizzo 1953a9e644cdSLuigi Rizzo while (budget) { 1954a9e644cdSLuigi Rizzo unsigned int rcnt = 0; 1955a9e644cdSLuigi Rizzo 1956a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 1957a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 1958a9e644cdSLuigi Rizzo 1959a9e644cdSLuigi Rizzo if (borrow) { 1960a9e644cdSLuigi Rizzo queue_budget = MIN(queue_budget, budget); 1961a9e644cdSLuigi Rizzo if (queue_budget == 0) { 1962a9e644cdSLuigi Rizzo break; 1963a9e644cdSLuigi Rizzo } 1964a9e644cdSLuigi Rizzo } 1965a9e644cdSLuigi Rizzo 1966a9e644cdSLuigi Rizzo if (i < sc->num_tx_rings) { 1967a9e644cdSLuigi Rizzo rcnt += ptnet_drain_transmit_queue(pq, 1968a9e644cdSLuigi Rizzo queue_budget, false); 1969a9e644cdSLuigi Rizzo } else { 1970a9e644cdSLuigi Rizzo rcnt += ptnet_rx_eof(pq, queue_budget, 1971a9e644cdSLuigi Rizzo false); 1972a9e644cdSLuigi Rizzo } 1973a9e644cdSLuigi Rizzo } 1974a9e644cdSLuigi Rizzo 1975a9e644cdSLuigi Rizzo if (!rcnt) { 1976a9e644cdSLuigi Rizzo /* A scan of the queues gave no result, we can 1977a9e644cdSLuigi Rizzo * stop here. */ 1978a9e644cdSLuigi Rizzo break; 1979a9e644cdSLuigi Rizzo } 1980a9e644cdSLuigi Rizzo 1981a9e644cdSLuigi Rizzo if (rcnt > budget) { 1982a9e644cdSLuigi Rizzo /* This may happen when initial budget < sc->num_rings, 1983a9e644cdSLuigi Rizzo * since one packet budget is given to each queue 1984a9e644cdSLuigi Rizzo * anyway. Just pretend we didn't eat "so much". */ 1985a9e644cdSLuigi Rizzo rcnt = budget; 1986a9e644cdSLuigi Rizzo } 1987a9e644cdSLuigi Rizzo count += rcnt; 1988a9e644cdSLuigi Rizzo budget -= rcnt; 1989a9e644cdSLuigi Rizzo borrow = true; 1990a9e644cdSLuigi Rizzo } 1991a9e644cdSLuigi Rizzo 1992a9e644cdSLuigi Rizzo 1993a9e644cdSLuigi Rizzo return count; 1994a9e644cdSLuigi Rizzo } 1995a9e644cdSLuigi Rizzo #endif /* DEVICE_POLLING */ 1996