1a9e644cdSLuigi Rizzo /*- 2a9e644cdSLuigi Rizzo * Copyright (c) 2016, Vincenzo Maffione 3a9e644cdSLuigi Rizzo * All rights reserved. 4a9e644cdSLuigi Rizzo * 5a9e644cdSLuigi Rizzo * Redistribution and use in source and binary forms, with or without 6a9e644cdSLuigi Rizzo * modification, are permitted provided that the following conditions 7a9e644cdSLuigi Rizzo * are met: 8a9e644cdSLuigi Rizzo * 1. Redistributions of source code must retain the above copyright 9a9e644cdSLuigi Rizzo * notice unmodified, this list of conditions, and the following 10a9e644cdSLuigi Rizzo * disclaimer. 11a9e644cdSLuigi Rizzo * 2. Redistributions in binary form must reproduce the above copyright 12a9e644cdSLuigi Rizzo * notice, this list of conditions and the following disclaimer in the 13a9e644cdSLuigi Rizzo * documentation and/or other materials provided with the distribution. 14a9e644cdSLuigi Rizzo * 15a9e644cdSLuigi Rizzo * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16a9e644cdSLuigi Rizzo * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17a9e644cdSLuigi Rizzo * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18a9e644cdSLuigi Rizzo * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19a9e644cdSLuigi Rizzo * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20a9e644cdSLuigi Rizzo * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21a9e644cdSLuigi Rizzo * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22a9e644cdSLuigi Rizzo * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23a9e644cdSLuigi Rizzo * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24a9e644cdSLuigi Rizzo * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25a9e644cdSLuigi Rizzo * 26a9e644cdSLuigi Rizzo * $FreeBSD$ 27a9e644cdSLuigi Rizzo */ 28a9e644cdSLuigi Rizzo 29a9e644cdSLuigi Rizzo /* Driver for ptnet paravirtualized network device. */ 30a9e644cdSLuigi Rizzo 31a9e644cdSLuigi Rizzo #include <sys/cdefs.h> 32a9e644cdSLuigi Rizzo 33a9e644cdSLuigi Rizzo #include <sys/types.h> 34a9e644cdSLuigi Rizzo #include <sys/param.h> 35a9e644cdSLuigi Rizzo #include <sys/systm.h> 36a9e644cdSLuigi Rizzo #include <sys/kernel.h> 37a9e644cdSLuigi Rizzo #include <sys/sockio.h> 38a9e644cdSLuigi Rizzo #include <sys/mbuf.h> 39a9e644cdSLuigi Rizzo #include <sys/malloc.h> 40a9e644cdSLuigi Rizzo #include <sys/module.h> 41a9e644cdSLuigi Rizzo #include <sys/socket.h> 42a9e644cdSLuigi Rizzo #include <sys/sysctl.h> 43a9e644cdSLuigi Rizzo #include <sys/lock.h> 44a9e644cdSLuigi Rizzo #include <sys/mutex.h> 45a9e644cdSLuigi Rizzo #include <sys/taskqueue.h> 46a9e644cdSLuigi Rizzo #include <sys/smp.h> 47a9e644cdSLuigi Rizzo #include <sys/time.h> 48a9e644cdSLuigi Rizzo #include <machine/smp.h> 49a9e644cdSLuigi Rizzo 50a9e644cdSLuigi Rizzo #include <vm/uma.h> 51a9e644cdSLuigi Rizzo #include <vm/vm.h> 52a9e644cdSLuigi Rizzo #include <vm/pmap.h> 53a9e644cdSLuigi Rizzo 54a9e644cdSLuigi Rizzo #include <net/ethernet.h> 55a9e644cdSLuigi Rizzo #include <net/if.h> 56a9e644cdSLuigi Rizzo #include <net/if_var.h> 57a9e644cdSLuigi Rizzo #include <net/if_arp.h> 58a9e644cdSLuigi Rizzo #include <net/if_dl.h> 59a9e644cdSLuigi Rizzo #include <net/if_types.h> 60a9e644cdSLuigi Rizzo #include <net/if_media.h> 61a9e644cdSLuigi Rizzo #include <net/if_vlan_var.h> 62a9e644cdSLuigi Rizzo #include <net/bpf.h> 63a9e644cdSLuigi Rizzo 64a9e644cdSLuigi Rizzo #include <netinet/in_systm.h> 65a9e644cdSLuigi Rizzo #include <netinet/in.h> 66a9e644cdSLuigi Rizzo #include <netinet/ip.h> 67a9e644cdSLuigi Rizzo #include <netinet/ip6.h> 68a9e644cdSLuigi Rizzo #include <netinet6/ip6_var.h> 69a9e644cdSLuigi Rizzo #include <netinet/udp.h> 70a9e644cdSLuigi Rizzo #include <netinet/tcp.h> 71a9e644cdSLuigi Rizzo 72a9e644cdSLuigi Rizzo #include <machine/bus.h> 73a9e644cdSLuigi Rizzo #include <machine/resource.h> 74a9e644cdSLuigi Rizzo #include <sys/bus.h> 75a9e644cdSLuigi Rizzo #include <sys/rman.h> 76a9e644cdSLuigi Rizzo 77a9e644cdSLuigi Rizzo #include <dev/pci/pcivar.h> 78a9e644cdSLuigi Rizzo #include <dev/pci/pcireg.h> 79a9e644cdSLuigi Rizzo 80a9e644cdSLuigi Rizzo #include "opt_inet.h" 81a9e644cdSLuigi Rizzo #include "opt_inet6.h" 82a9e644cdSLuigi Rizzo 83a9e644cdSLuigi Rizzo #include <sys/selinfo.h> 84a9e644cdSLuigi Rizzo #include <net/netmap.h> 85a9e644cdSLuigi Rizzo #include <dev/netmap/netmap_kern.h> 86a9e644cdSLuigi Rizzo #include <net/netmap_virt.h> 87a9e644cdSLuigi Rizzo #include <dev/netmap/netmap_mem2.h> 88a9e644cdSLuigi Rizzo #include <dev/virtio/network/virtio_net.h> 89a9e644cdSLuigi Rizzo 90*2ec213abSVincenzo Maffione #ifdef WITH_PTNETMAP 91*2ec213abSVincenzo Maffione 92a9e644cdSLuigi Rizzo #ifndef INET 93a9e644cdSLuigi Rizzo #error "INET not defined, cannot support offloadings" 94a9e644cdSLuigi Rizzo #endif 95a9e644cdSLuigi Rizzo 96a9e644cdSLuigi Rizzo #if __FreeBSD_version >= 1100000 97a9e644cdSLuigi Rizzo static uint64_t ptnet_get_counter(if_t, ift_counter); 98a9e644cdSLuigi Rizzo #else 99a9e644cdSLuigi Rizzo typedef struct ifnet *if_t; 100a9e644cdSLuigi Rizzo #define if_getsoftc(_ifp) (_ifp)->if_softc 101a9e644cdSLuigi Rizzo #endif 102a9e644cdSLuigi Rizzo 103a9e644cdSLuigi Rizzo //#define PTNETMAP_STATS 104a9e644cdSLuigi Rizzo //#define DEBUG 105a9e644cdSLuigi Rizzo #ifdef DEBUG 106a9e644cdSLuigi Rizzo #define DBG(x) x 107a9e644cdSLuigi Rizzo #else /* !DEBUG */ 108a9e644cdSLuigi Rizzo #define DBG(x) 109a9e644cdSLuigi Rizzo #endif /* !DEBUG */ 110a9e644cdSLuigi Rizzo 111a9e644cdSLuigi Rizzo extern int ptnet_vnet_hdr; /* Tunable parameter */ 112a9e644cdSLuigi Rizzo 113a9e644cdSLuigi Rizzo struct ptnet_softc; 114a9e644cdSLuigi Rizzo 115a9e644cdSLuigi Rizzo struct ptnet_queue_stats { 116a9e644cdSLuigi Rizzo uint64_t packets; /* if_[io]packets */ 117a9e644cdSLuigi Rizzo uint64_t bytes; /* if_[io]bytes */ 118a9e644cdSLuigi Rizzo uint64_t errors; /* if_[io]errors */ 119a9e644cdSLuigi Rizzo uint64_t iqdrops; /* if_iqdrops */ 120a9e644cdSLuigi Rizzo uint64_t mcasts; /* if_[io]mcasts */ 121a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS 122a9e644cdSLuigi Rizzo uint64_t intrs; 123a9e644cdSLuigi Rizzo uint64_t kicks; 124a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */ 125a9e644cdSLuigi Rizzo }; 126a9e644cdSLuigi Rizzo 127a9e644cdSLuigi Rizzo struct ptnet_queue { 128a9e644cdSLuigi Rizzo struct ptnet_softc *sc; 129a9e644cdSLuigi Rizzo struct resource *irq; 130a9e644cdSLuigi Rizzo void *cookie; 131a9e644cdSLuigi Rizzo int kring_id; 132b6e66be2SVincenzo Maffione struct nm_csb_atok *atok; 133b6e66be2SVincenzo Maffione struct nm_csb_ktoa *ktoa; 134a9e644cdSLuigi Rizzo unsigned int kick; 135a9e644cdSLuigi Rizzo struct mtx lock; 136a9e644cdSLuigi Rizzo struct buf_ring *bufring; /* for TX queues */ 137a9e644cdSLuigi Rizzo struct ptnet_queue_stats stats; 138a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS 139a9e644cdSLuigi Rizzo struct ptnet_queue_stats last_stats; 140a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */ 141a9e644cdSLuigi Rizzo struct taskqueue *taskq; 142a9e644cdSLuigi Rizzo struct task task; 143a9e644cdSLuigi Rizzo char lock_name[16]; 144a9e644cdSLuigi Rizzo }; 145a9e644cdSLuigi Rizzo 146a9e644cdSLuigi Rizzo #define PTNET_Q_LOCK(_pq) mtx_lock(&(_pq)->lock) 147a9e644cdSLuigi Rizzo #define PTNET_Q_TRYLOCK(_pq) mtx_trylock(&(_pq)->lock) 148a9e644cdSLuigi Rizzo #define PTNET_Q_UNLOCK(_pq) mtx_unlock(&(_pq)->lock) 149a9e644cdSLuigi Rizzo 150a9e644cdSLuigi Rizzo struct ptnet_softc { 151a9e644cdSLuigi Rizzo device_t dev; 152a9e644cdSLuigi Rizzo if_t ifp; 153a9e644cdSLuigi Rizzo struct ifmedia media; 154a9e644cdSLuigi Rizzo struct mtx lock; 155a9e644cdSLuigi Rizzo char lock_name[16]; 156a9e644cdSLuigi Rizzo char hwaddr[ETHER_ADDR_LEN]; 157a9e644cdSLuigi Rizzo 158a9e644cdSLuigi Rizzo /* Mirror of PTFEAT register. */ 159a9e644cdSLuigi Rizzo uint32_t ptfeatures; 160a9e644cdSLuigi Rizzo unsigned int vnet_hdr_len; 161a9e644cdSLuigi Rizzo 162a9e644cdSLuigi Rizzo /* PCI BARs support. */ 163a9e644cdSLuigi Rizzo struct resource *iomem; 164a9e644cdSLuigi Rizzo struct resource *msix_mem; 165a9e644cdSLuigi Rizzo 166a9e644cdSLuigi Rizzo unsigned int num_rings; 167a9e644cdSLuigi Rizzo unsigned int num_tx_rings; 168a9e644cdSLuigi Rizzo struct ptnet_queue *queues; 169a9e644cdSLuigi Rizzo struct ptnet_queue *rxqueues; 170b6e66be2SVincenzo Maffione struct nm_csb_atok *csb_gh; 171b6e66be2SVincenzo Maffione struct nm_csb_ktoa *csb_hg; 172a9e644cdSLuigi Rizzo 173a9e644cdSLuigi Rizzo unsigned int min_tx_space; 174a9e644cdSLuigi Rizzo 175a9e644cdSLuigi Rizzo struct netmap_pt_guest_adapter *ptna; 176a9e644cdSLuigi Rizzo 177a9e644cdSLuigi Rizzo struct callout tick; 178a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS 179a9e644cdSLuigi Rizzo struct timeval last_ts; 180a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */ 181a9e644cdSLuigi Rizzo }; 182a9e644cdSLuigi Rizzo 183a9e644cdSLuigi Rizzo #define PTNET_CORE_LOCK(_sc) mtx_lock(&(_sc)->lock) 184a9e644cdSLuigi Rizzo #define PTNET_CORE_UNLOCK(_sc) mtx_unlock(&(_sc)->lock) 185a9e644cdSLuigi Rizzo 186a9e644cdSLuigi Rizzo static int ptnet_probe(device_t); 187a9e644cdSLuigi Rizzo static int ptnet_attach(device_t); 188a9e644cdSLuigi Rizzo static int ptnet_detach(device_t); 189a9e644cdSLuigi Rizzo static int ptnet_suspend(device_t); 190a9e644cdSLuigi Rizzo static int ptnet_resume(device_t); 191a9e644cdSLuigi Rizzo static int ptnet_shutdown(device_t); 192a9e644cdSLuigi Rizzo 193a9e644cdSLuigi Rizzo static void ptnet_init(void *opaque); 194a9e644cdSLuigi Rizzo static int ptnet_ioctl(if_t ifp, u_long cmd, caddr_t data); 195a9e644cdSLuigi Rizzo static int ptnet_init_locked(struct ptnet_softc *sc); 196a9e644cdSLuigi Rizzo static int ptnet_stop(struct ptnet_softc *sc); 197a9e644cdSLuigi Rizzo static int ptnet_transmit(if_t ifp, struct mbuf *m); 198a9e644cdSLuigi Rizzo static int ptnet_drain_transmit_queue(struct ptnet_queue *pq, 199a9e644cdSLuigi Rizzo unsigned int budget, 200a9e644cdSLuigi Rizzo bool may_resched); 201a9e644cdSLuigi Rizzo static void ptnet_qflush(if_t ifp); 202a9e644cdSLuigi Rizzo static void ptnet_tx_task(void *context, int pending); 203a9e644cdSLuigi Rizzo 204a9e644cdSLuigi Rizzo static int ptnet_media_change(if_t ifp); 205a9e644cdSLuigi Rizzo static void ptnet_media_status(if_t ifp, struct ifmediareq *ifmr); 206a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS 207a9e644cdSLuigi Rizzo static void ptnet_tick(void *opaque); 208a9e644cdSLuigi Rizzo #endif 209a9e644cdSLuigi Rizzo 210a9e644cdSLuigi Rizzo static int ptnet_irqs_init(struct ptnet_softc *sc); 211a9e644cdSLuigi Rizzo static void ptnet_irqs_fini(struct ptnet_softc *sc); 212a9e644cdSLuigi Rizzo 213b6e66be2SVincenzo Maffione static uint32_t ptnet_nm_ptctl(struct ptnet_softc *sc, uint32_t cmd); 2142ff91c17SVincenzo Maffione static int ptnet_nm_config(struct netmap_adapter *na, 2152ff91c17SVincenzo Maffione struct nm_config_info *info); 216a9e644cdSLuigi Rizzo static void ptnet_update_vnet_hdr(struct ptnet_softc *sc); 217a9e644cdSLuigi Rizzo static int ptnet_nm_register(struct netmap_adapter *na, int onoff); 218a9e644cdSLuigi Rizzo static int ptnet_nm_txsync(struct netmap_kring *kring, int flags); 219a9e644cdSLuigi Rizzo static int ptnet_nm_rxsync(struct netmap_kring *kring, int flags); 2204f80b14cSVincenzo Maffione static void ptnet_nm_intr(struct netmap_adapter *na, int onoff); 221a9e644cdSLuigi Rizzo 222a9e644cdSLuigi Rizzo static void ptnet_tx_intr(void *opaque); 223a9e644cdSLuigi Rizzo static void ptnet_rx_intr(void *opaque); 224a9e644cdSLuigi Rizzo 225a9e644cdSLuigi Rizzo static unsigned ptnet_rx_discard(struct netmap_kring *kring, 226a9e644cdSLuigi Rizzo unsigned int head); 227a9e644cdSLuigi Rizzo static int ptnet_rx_eof(struct ptnet_queue *pq, unsigned int budget, 228a9e644cdSLuigi Rizzo bool may_resched); 229a9e644cdSLuigi Rizzo static void ptnet_rx_task(void *context, int pending); 230a9e644cdSLuigi Rizzo 231a9e644cdSLuigi Rizzo #ifdef DEVICE_POLLING 232a9e644cdSLuigi Rizzo static poll_handler_t ptnet_poll; 233a9e644cdSLuigi Rizzo #endif 234a9e644cdSLuigi Rizzo 235a9e644cdSLuigi Rizzo static device_method_t ptnet_methods[] = { 236a9e644cdSLuigi Rizzo DEVMETHOD(device_probe, ptnet_probe), 237a9e644cdSLuigi Rizzo DEVMETHOD(device_attach, ptnet_attach), 238a9e644cdSLuigi Rizzo DEVMETHOD(device_detach, ptnet_detach), 239a9e644cdSLuigi Rizzo DEVMETHOD(device_suspend, ptnet_suspend), 240a9e644cdSLuigi Rizzo DEVMETHOD(device_resume, ptnet_resume), 241a9e644cdSLuigi Rizzo DEVMETHOD(device_shutdown, ptnet_shutdown), 242a9e644cdSLuigi Rizzo DEVMETHOD_END 243a9e644cdSLuigi Rizzo }; 244a9e644cdSLuigi Rizzo 245a9e644cdSLuigi Rizzo static driver_t ptnet_driver = { 246a9e644cdSLuigi Rizzo "ptnet", 247a9e644cdSLuigi Rizzo ptnet_methods, 248a9e644cdSLuigi Rizzo sizeof(struct ptnet_softc) 249a9e644cdSLuigi Rizzo }; 250a9e644cdSLuigi Rizzo 251a9e644cdSLuigi Rizzo /* We use (SI_ORDER_MIDDLE+2) here, see DEV_MODULE_ORDERED() invocation. */ 252a9e644cdSLuigi Rizzo static devclass_t ptnet_devclass; 253a9e644cdSLuigi Rizzo DRIVER_MODULE_ORDERED(ptnet, pci, ptnet_driver, ptnet_devclass, 254a9e644cdSLuigi Rizzo NULL, NULL, SI_ORDER_MIDDLE + 2); 255a9e644cdSLuigi Rizzo 256a9e644cdSLuigi Rizzo static int 257a9e644cdSLuigi Rizzo ptnet_probe(device_t dev) 258a9e644cdSLuigi Rizzo { 259a9e644cdSLuigi Rizzo if (pci_get_vendor(dev) != PTNETMAP_PCI_VENDOR_ID || 260a9e644cdSLuigi Rizzo pci_get_device(dev) != PTNETMAP_PCI_NETIF_ID) { 261a9e644cdSLuigi Rizzo return (ENXIO); 262a9e644cdSLuigi Rizzo } 263a9e644cdSLuigi Rizzo 264a9e644cdSLuigi Rizzo device_set_desc(dev, "ptnet network adapter"); 265a9e644cdSLuigi Rizzo 266a9e644cdSLuigi Rizzo return (BUS_PROBE_DEFAULT); 267a9e644cdSLuigi Rizzo } 268a9e644cdSLuigi Rizzo 269a9e644cdSLuigi Rizzo static inline void ptnet_kick(struct ptnet_queue *pq) 270a9e644cdSLuigi Rizzo { 271a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS 272a9e644cdSLuigi Rizzo pq->stats.kicks ++; 273a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */ 274a9e644cdSLuigi Rizzo bus_write_4(pq->sc->iomem, pq->kick, 0); 275a9e644cdSLuigi Rizzo } 276a9e644cdSLuigi Rizzo 277a9e644cdSLuigi Rizzo #define PTNET_BUF_RING_SIZE 4096 278a9e644cdSLuigi Rizzo #define PTNET_RX_BUDGET 512 279a9e644cdSLuigi Rizzo #define PTNET_RX_BATCH 1 280a9e644cdSLuigi Rizzo #define PTNET_TX_BUDGET 512 281a9e644cdSLuigi Rizzo #define PTNET_TX_BATCH 64 282a9e644cdSLuigi Rizzo #define PTNET_HDR_SIZE sizeof(struct virtio_net_hdr_mrg_rxbuf) 283a9e644cdSLuigi Rizzo #define PTNET_MAX_PKT_SIZE 65536 284a9e644cdSLuigi Rizzo 285d337c8c7SVincenzo Maffione #define PTNET_CSUM_OFFLOAD (CSUM_TCP | CSUM_UDP) 286d337c8c7SVincenzo Maffione #define PTNET_CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6 | CSUM_UDP_IPV6) 287a9e644cdSLuigi Rizzo #define PTNET_ALL_OFFLOAD (CSUM_TSO | PTNET_CSUM_OFFLOAD |\ 288a9e644cdSLuigi Rizzo PTNET_CSUM_OFFLOAD_IPV6) 289a9e644cdSLuigi Rizzo 290a9e644cdSLuigi Rizzo static int 291a9e644cdSLuigi Rizzo ptnet_attach(device_t dev) 292a9e644cdSLuigi Rizzo { 293844a6f0cSLuigi Rizzo uint32_t ptfeatures = 0; 294a9e644cdSLuigi Rizzo unsigned int num_rx_rings, num_tx_rings; 295a9e644cdSLuigi Rizzo struct netmap_adapter na_arg; 296a9e644cdSLuigi Rizzo unsigned int nifp_offset; 297a9e644cdSLuigi Rizzo struct ptnet_softc *sc; 298a9e644cdSLuigi Rizzo if_t ifp; 299a9e644cdSLuigi Rizzo uint32_t macreg; 300a9e644cdSLuigi Rizzo int err, rid; 301a9e644cdSLuigi Rizzo int i; 302a9e644cdSLuigi Rizzo 303a9e644cdSLuigi Rizzo sc = device_get_softc(dev); 304a9e644cdSLuigi Rizzo sc->dev = dev; 305a9e644cdSLuigi Rizzo 306a9e644cdSLuigi Rizzo /* Setup PCI resources. */ 307a9e644cdSLuigi Rizzo pci_enable_busmaster(dev); 308a9e644cdSLuigi Rizzo 309a9e644cdSLuigi Rizzo rid = PCIR_BAR(PTNETMAP_IO_PCI_BAR); 310a9e644cdSLuigi Rizzo sc->iomem = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid, 311a9e644cdSLuigi Rizzo RF_ACTIVE); 312a9e644cdSLuigi Rizzo if (sc->iomem == NULL) { 313a9e644cdSLuigi Rizzo device_printf(dev, "Failed to map I/O BAR\n"); 314a9e644cdSLuigi Rizzo return (ENXIO); 315a9e644cdSLuigi Rizzo } 316a9e644cdSLuigi Rizzo 317844a6f0cSLuigi Rizzo /* Negotiate features with the hypervisor. */ 318a9e644cdSLuigi Rizzo if (ptnet_vnet_hdr) { 319a9e644cdSLuigi Rizzo ptfeatures |= PTNETMAP_F_VNET_HDR; 320a9e644cdSLuigi Rizzo } 321a9e644cdSLuigi Rizzo bus_write_4(sc->iomem, PTNET_IO_PTFEAT, ptfeatures); /* wanted */ 322a9e644cdSLuigi Rizzo ptfeatures = bus_read_4(sc->iomem, PTNET_IO_PTFEAT); /* acked */ 323a9e644cdSLuigi Rizzo sc->ptfeatures = ptfeatures; 324a9e644cdSLuigi Rizzo 32546023447SVincenzo Maffione num_tx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_RINGS); 32646023447SVincenzo Maffione num_rx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_RINGS); 32746023447SVincenzo Maffione sc->num_rings = num_tx_rings + num_rx_rings; 32846023447SVincenzo Maffione sc->num_tx_rings = num_tx_rings; 32946023447SVincenzo Maffione 330b6e66be2SVincenzo Maffione if (sc->num_rings * sizeof(struct nm_csb_atok) > PAGE_SIZE) { 33146023447SVincenzo Maffione device_printf(dev, "CSB cannot handle that many rings (%u)\n", 33246023447SVincenzo Maffione sc->num_rings); 33346023447SVincenzo Maffione err = ENOMEM; 33446023447SVincenzo Maffione goto err_path; 33546023447SVincenzo Maffione } 33646023447SVincenzo Maffione 33746023447SVincenzo Maffione /* Allocate CSB and carry out CSB allocation protocol. */ 33846023447SVincenzo Maffione sc->csb_gh = contigmalloc(2*PAGE_SIZE, M_DEVBUF, M_NOWAIT | M_ZERO, 33946023447SVincenzo Maffione (size_t)0, -1UL, PAGE_SIZE, 0); 34046023447SVincenzo Maffione if (sc->csb_gh == NULL) { 341a9e644cdSLuigi Rizzo device_printf(dev, "Failed to allocate CSB\n"); 342a9e644cdSLuigi Rizzo err = ENOMEM; 343a9e644cdSLuigi Rizzo goto err_path; 344a9e644cdSLuigi Rizzo } 345b6e66be2SVincenzo Maffione sc->csb_hg = (struct nm_csb_ktoa *)(((char *)sc->csb_gh) + PAGE_SIZE); 346a9e644cdSLuigi Rizzo 347a9e644cdSLuigi Rizzo { 348a2a74091SLuigi Rizzo /* 349a2a74091SLuigi Rizzo * We use uint64_t rather than vm_paddr_t since we 350a2a74091SLuigi Rizzo * need 64 bit addresses even on 32 bit platforms. 351a2a74091SLuigi Rizzo */ 35246023447SVincenzo Maffione uint64_t paddr = vtophys(sc->csb_gh); 353a9e644cdSLuigi Rizzo 35446023447SVincenzo Maffione /* CSB allocation protocol: write to BAH first, then 35546023447SVincenzo Maffione * to BAL (for both GH and HG sections). */ 35646023447SVincenzo Maffione bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAH, 357a9e644cdSLuigi Rizzo (paddr >> 32) & 0xffffffff); 35846023447SVincenzo Maffione bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAL, 35946023447SVincenzo Maffione paddr & 0xffffffff); 36046023447SVincenzo Maffione paddr = vtophys(sc->csb_hg); 36146023447SVincenzo Maffione bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAH, 36246023447SVincenzo Maffione (paddr >> 32) & 0xffffffff); 36346023447SVincenzo Maffione bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAL, 36446023447SVincenzo Maffione paddr & 0xffffffff); 365a9e644cdSLuigi Rizzo } 366a9e644cdSLuigi Rizzo 367a9e644cdSLuigi Rizzo /* Allocate and initialize per-queue data structures. */ 368ac2fffa4SPedro F. Giffuni sc->queues = malloc(sizeof(struct ptnet_queue) * sc->num_rings, 369a9e644cdSLuigi Rizzo M_DEVBUF, M_NOWAIT | M_ZERO); 370a9e644cdSLuigi Rizzo if (sc->queues == NULL) { 371a9e644cdSLuigi Rizzo err = ENOMEM; 372a9e644cdSLuigi Rizzo goto err_path; 373a9e644cdSLuigi Rizzo } 374a9e644cdSLuigi Rizzo sc->rxqueues = sc->queues + num_tx_rings; 375a9e644cdSLuigi Rizzo 376a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 377a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 378a9e644cdSLuigi Rizzo 379a9e644cdSLuigi Rizzo pq->sc = sc; 380a9e644cdSLuigi Rizzo pq->kring_id = i; 381a9e644cdSLuigi Rizzo pq->kick = PTNET_IO_KICK_BASE + 4 * i; 382b6e66be2SVincenzo Maffione pq->atok = sc->csb_gh + i; 383b6e66be2SVincenzo Maffione pq->ktoa = sc->csb_hg + i; 384a9e644cdSLuigi Rizzo snprintf(pq->lock_name, sizeof(pq->lock_name), "%s-%d", 385a9e644cdSLuigi Rizzo device_get_nameunit(dev), i); 386a9e644cdSLuigi Rizzo mtx_init(&pq->lock, pq->lock_name, NULL, MTX_DEF); 387a9e644cdSLuigi Rizzo if (i >= num_tx_rings) { 388a9e644cdSLuigi Rizzo /* RX queue: fix kring_id. */ 389a9e644cdSLuigi Rizzo pq->kring_id -= num_tx_rings; 390a9e644cdSLuigi Rizzo } else { 391a9e644cdSLuigi Rizzo /* TX queue: allocate buf_ring. */ 392a9e644cdSLuigi Rizzo pq->bufring = buf_ring_alloc(PTNET_BUF_RING_SIZE, 393a9e644cdSLuigi Rizzo M_DEVBUF, M_NOWAIT, &pq->lock); 394a9e644cdSLuigi Rizzo if (pq->bufring == NULL) { 395a9e644cdSLuigi Rizzo err = ENOMEM; 396a9e644cdSLuigi Rizzo goto err_path; 397a9e644cdSLuigi Rizzo } 398a9e644cdSLuigi Rizzo } 399a9e644cdSLuigi Rizzo } 400a9e644cdSLuigi Rizzo 401a9e644cdSLuigi Rizzo sc->min_tx_space = 64; /* Safe initial value. */ 402a9e644cdSLuigi Rizzo 403a9e644cdSLuigi Rizzo err = ptnet_irqs_init(sc); 404a9e644cdSLuigi Rizzo if (err) { 405a9e644cdSLuigi Rizzo goto err_path; 406a9e644cdSLuigi Rizzo } 407a9e644cdSLuigi Rizzo 408a9e644cdSLuigi Rizzo /* Setup Ethernet interface. */ 409a9e644cdSLuigi Rizzo sc->ifp = ifp = if_alloc(IFT_ETHER); 410a9e644cdSLuigi Rizzo if (ifp == NULL) { 411a9e644cdSLuigi Rizzo device_printf(dev, "Failed to allocate ifnet\n"); 412a9e644cdSLuigi Rizzo err = ENOMEM; 413a9e644cdSLuigi Rizzo goto err_path; 414a9e644cdSLuigi Rizzo } 415a9e644cdSLuigi Rizzo 416a9e644cdSLuigi Rizzo if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 417a9e644cdSLuigi Rizzo ifp->if_baudrate = IF_Gbps(10); 418a9e644cdSLuigi Rizzo ifp->if_softc = sc; 419a9e644cdSLuigi Rizzo ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX; 420a9e644cdSLuigi Rizzo ifp->if_init = ptnet_init; 421a9e644cdSLuigi Rizzo ifp->if_ioctl = ptnet_ioctl; 422a9e644cdSLuigi Rizzo #if __FreeBSD_version >= 1100000 423a9e644cdSLuigi Rizzo ifp->if_get_counter = ptnet_get_counter; 424a9e644cdSLuigi Rizzo #endif 425a9e644cdSLuigi Rizzo ifp->if_transmit = ptnet_transmit; 426a9e644cdSLuigi Rizzo ifp->if_qflush = ptnet_qflush; 427a9e644cdSLuigi Rizzo 428a9e644cdSLuigi Rizzo ifmedia_init(&sc->media, IFM_IMASK, ptnet_media_change, 429a9e644cdSLuigi Rizzo ptnet_media_status); 430a9e644cdSLuigi Rizzo ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX, 0, NULL); 431a9e644cdSLuigi Rizzo ifmedia_set(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX); 432a9e644cdSLuigi Rizzo 433a9e644cdSLuigi Rizzo macreg = bus_read_4(sc->iomem, PTNET_IO_MAC_HI); 434a9e644cdSLuigi Rizzo sc->hwaddr[0] = (macreg >> 8) & 0xff; 435a9e644cdSLuigi Rizzo sc->hwaddr[1] = macreg & 0xff; 436a9e644cdSLuigi Rizzo macreg = bus_read_4(sc->iomem, PTNET_IO_MAC_LO); 437a9e644cdSLuigi Rizzo sc->hwaddr[2] = (macreg >> 24) & 0xff; 438a9e644cdSLuigi Rizzo sc->hwaddr[3] = (macreg >> 16) & 0xff; 439a9e644cdSLuigi Rizzo sc->hwaddr[4] = (macreg >> 8) & 0xff; 440a9e644cdSLuigi Rizzo sc->hwaddr[5] = macreg & 0xff; 441a9e644cdSLuigi Rizzo 442a9e644cdSLuigi Rizzo ether_ifattach(ifp, sc->hwaddr); 443a9e644cdSLuigi Rizzo 444a9e644cdSLuigi Rizzo ifp->if_hdrlen = sizeof(struct ether_vlan_header); 445a9e644cdSLuigi Rizzo ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU; 446a9e644cdSLuigi Rizzo 447a9e644cdSLuigi Rizzo if (sc->ptfeatures & PTNETMAP_F_VNET_HDR) { 448a9e644cdSLuigi Rizzo /* Similarly to what the vtnet driver does, we can emulate 449a9e644cdSLuigi Rizzo * VLAN offloadings by inserting and removing the 802.1Q 450a9e644cdSLuigi Rizzo * header during transmit and receive. We are then able 451a9e644cdSLuigi Rizzo * to do checksum offloading of VLAN frames. */ 452a9e644cdSLuigi Rizzo ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 453a9e644cdSLuigi Rizzo | IFCAP_VLAN_HWCSUM 454a9e644cdSLuigi Rizzo | IFCAP_TSO | IFCAP_LRO 455a9e644cdSLuigi Rizzo | IFCAP_VLAN_HWTSO 456a9e644cdSLuigi Rizzo | IFCAP_VLAN_HWTAGGING; 457a9e644cdSLuigi Rizzo } 458a9e644cdSLuigi Rizzo 459a9e644cdSLuigi Rizzo ifp->if_capenable = ifp->if_capabilities; 460a9e644cdSLuigi Rizzo #ifdef DEVICE_POLLING 461a9e644cdSLuigi Rizzo /* Don't enable polling by default. */ 462a9e644cdSLuigi Rizzo ifp->if_capabilities |= IFCAP_POLLING; 463a9e644cdSLuigi Rizzo #endif 464a9e644cdSLuigi Rizzo snprintf(sc->lock_name, sizeof(sc->lock_name), 465a9e644cdSLuigi Rizzo "%s", device_get_nameunit(dev)); 466a9e644cdSLuigi Rizzo mtx_init(&sc->lock, sc->lock_name, "ptnet core lock", MTX_DEF); 467a9e644cdSLuigi Rizzo callout_init_mtx(&sc->tick, &sc->lock, 0); 468a9e644cdSLuigi Rizzo 469a9e644cdSLuigi Rizzo /* Prepare a netmap_adapter struct instance to do netmap_attach(). */ 470a9e644cdSLuigi Rizzo nifp_offset = bus_read_4(sc->iomem, PTNET_IO_NIFP_OFS); 471a9e644cdSLuigi Rizzo memset(&na_arg, 0, sizeof(na_arg)); 472a9e644cdSLuigi Rizzo na_arg.ifp = ifp; 473a9e644cdSLuigi Rizzo na_arg.num_tx_desc = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_SLOTS); 474a9e644cdSLuigi Rizzo na_arg.num_rx_desc = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_SLOTS); 475a9e644cdSLuigi Rizzo na_arg.num_tx_rings = num_tx_rings; 476a9e644cdSLuigi Rizzo na_arg.num_rx_rings = num_rx_rings; 477a9e644cdSLuigi Rizzo na_arg.nm_config = ptnet_nm_config; 478a9e644cdSLuigi Rizzo na_arg.nm_krings_create = ptnet_nm_krings_create; 479a9e644cdSLuigi Rizzo na_arg.nm_krings_delete = ptnet_nm_krings_delete; 480a9e644cdSLuigi Rizzo na_arg.nm_dtor = ptnet_nm_dtor; 4814f80b14cSVincenzo Maffione na_arg.nm_intr = ptnet_nm_intr; 482a9e644cdSLuigi Rizzo na_arg.nm_register = ptnet_nm_register; 483a9e644cdSLuigi Rizzo na_arg.nm_txsync = ptnet_nm_txsync; 484a9e644cdSLuigi Rizzo na_arg.nm_rxsync = ptnet_nm_rxsync; 485a9e644cdSLuigi Rizzo 48646023447SVincenzo Maffione netmap_pt_guest_attach(&na_arg, nifp_offset, 487844a6f0cSLuigi Rizzo bus_read_4(sc->iomem, PTNET_IO_HOSTMEMID)); 488a9e644cdSLuigi Rizzo 489a9e644cdSLuigi Rizzo /* Now a netmap adapter for this ifp has been allocated, and it 490a9e644cdSLuigi Rizzo * can be accessed through NA(ifp). We also have to initialize the CSB 491a9e644cdSLuigi Rizzo * pointer. */ 492a9e644cdSLuigi Rizzo sc->ptna = (struct netmap_pt_guest_adapter *)NA(ifp); 493a9e644cdSLuigi Rizzo 494a9e644cdSLuigi Rizzo /* If virtio-net header was negotiated, set the virt_hdr_len field in 495a9e644cdSLuigi Rizzo * the netmap adapter, to inform users that this netmap adapter requires 496a9e644cdSLuigi Rizzo * the application to deal with the headers. */ 497a9e644cdSLuigi Rizzo ptnet_update_vnet_hdr(sc); 498a9e644cdSLuigi Rizzo 499a9e644cdSLuigi Rizzo device_printf(dev, "%s() completed\n", __func__); 500a9e644cdSLuigi Rizzo 501a9e644cdSLuigi Rizzo return (0); 502a9e644cdSLuigi Rizzo 503a9e644cdSLuigi Rizzo err_path: 504a9e644cdSLuigi Rizzo ptnet_detach(dev); 505a9e644cdSLuigi Rizzo return err; 506a9e644cdSLuigi Rizzo } 507a9e644cdSLuigi Rizzo 508b6e66be2SVincenzo Maffione /* Stop host sync-kloop if it was running. */ 509b6e66be2SVincenzo Maffione static void 510b6e66be2SVincenzo Maffione ptnet_device_shutdown(struct ptnet_softc *sc) 511b6e66be2SVincenzo Maffione { 512b6e66be2SVincenzo Maffione ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_DELETE); 513b6e66be2SVincenzo Maffione bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAH, 0); 514b6e66be2SVincenzo Maffione bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAL, 0); 515b6e66be2SVincenzo Maffione bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAH, 0); 516b6e66be2SVincenzo Maffione bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAL, 0); 517b6e66be2SVincenzo Maffione } 518b6e66be2SVincenzo Maffione 519a9e644cdSLuigi Rizzo static int 520a9e644cdSLuigi Rizzo ptnet_detach(device_t dev) 521a9e644cdSLuigi Rizzo { 522a9e644cdSLuigi Rizzo struct ptnet_softc *sc = device_get_softc(dev); 523a9e644cdSLuigi Rizzo int i; 524a9e644cdSLuigi Rizzo 525b6e66be2SVincenzo Maffione ptnet_device_shutdown(sc); 526b6e66be2SVincenzo Maffione 527a9e644cdSLuigi Rizzo #ifdef DEVICE_POLLING 528a9e644cdSLuigi Rizzo if (sc->ifp->if_capenable & IFCAP_POLLING) { 529a9e644cdSLuigi Rizzo ether_poll_deregister(sc->ifp); 530a9e644cdSLuigi Rizzo } 531a9e644cdSLuigi Rizzo #endif 532a9e644cdSLuigi Rizzo callout_drain(&sc->tick); 533a9e644cdSLuigi Rizzo 534a9e644cdSLuigi Rizzo if (sc->queues) { 535a9e644cdSLuigi Rizzo /* Drain taskqueues before calling if_detach. */ 536a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 537a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 538a9e644cdSLuigi Rizzo 539a9e644cdSLuigi Rizzo if (pq->taskq) { 540a9e644cdSLuigi Rizzo taskqueue_drain(pq->taskq, &pq->task); 541a9e644cdSLuigi Rizzo } 542a9e644cdSLuigi Rizzo } 543a9e644cdSLuigi Rizzo } 544a9e644cdSLuigi Rizzo 545a9e644cdSLuigi Rizzo if (sc->ifp) { 546a9e644cdSLuigi Rizzo ether_ifdetach(sc->ifp); 547a9e644cdSLuigi Rizzo 548a9e644cdSLuigi Rizzo /* Uninitialize netmap adapters for this device. */ 549a9e644cdSLuigi Rizzo netmap_detach(sc->ifp); 550a9e644cdSLuigi Rizzo 551a9e644cdSLuigi Rizzo ifmedia_removeall(&sc->media); 552a9e644cdSLuigi Rizzo if_free(sc->ifp); 553a9e644cdSLuigi Rizzo sc->ifp = NULL; 554a9e644cdSLuigi Rizzo } 555a9e644cdSLuigi Rizzo 556a9e644cdSLuigi Rizzo ptnet_irqs_fini(sc); 557a9e644cdSLuigi Rizzo 55846023447SVincenzo Maffione if (sc->csb_gh) { 55946023447SVincenzo Maffione contigfree(sc->csb_gh, 2*PAGE_SIZE, M_DEVBUF); 56046023447SVincenzo Maffione sc->csb_gh = NULL; 56146023447SVincenzo Maffione sc->csb_hg = NULL; 562a9e644cdSLuigi Rizzo } 563a9e644cdSLuigi Rizzo 564a9e644cdSLuigi Rizzo if (sc->queues) { 565a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 566a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 567a9e644cdSLuigi Rizzo 568a9e644cdSLuigi Rizzo if (mtx_initialized(&pq->lock)) { 569a9e644cdSLuigi Rizzo mtx_destroy(&pq->lock); 570a9e644cdSLuigi Rizzo } 571a9e644cdSLuigi Rizzo if (pq->bufring != NULL) { 572a9e644cdSLuigi Rizzo buf_ring_free(pq->bufring, M_DEVBUF); 573a9e644cdSLuigi Rizzo } 574a9e644cdSLuigi Rizzo } 575a9e644cdSLuigi Rizzo free(sc->queues, M_DEVBUF); 576a9e644cdSLuigi Rizzo sc->queues = NULL; 577a9e644cdSLuigi Rizzo } 578a9e644cdSLuigi Rizzo 579a9e644cdSLuigi Rizzo if (sc->iomem) { 580a9e644cdSLuigi Rizzo bus_release_resource(dev, SYS_RES_IOPORT, 581a9e644cdSLuigi Rizzo PCIR_BAR(PTNETMAP_IO_PCI_BAR), sc->iomem); 582a9e644cdSLuigi Rizzo sc->iomem = NULL; 583a9e644cdSLuigi Rizzo } 584a9e644cdSLuigi Rizzo 585a9e644cdSLuigi Rizzo mtx_destroy(&sc->lock); 586a9e644cdSLuigi Rizzo 587a9e644cdSLuigi Rizzo device_printf(dev, "%s() completed\n", __func__); 588a9e644cdSLuigi Rizzo 589a9e644cdSLuigi Rizzo return (0); 590a9e644cdSLuigi Rizzo } 591a9e644cdSLuigi Rizzo 592a9e644cdSLuigi Rizzo static int 593a9e644cdSLuigi Rizzo ptnet_suspend(device_t dev) 594a9e644cdSLuigi Rizzo { 595b6e66be2SVincenzo Maffione struct ptnet_softc *sc = device_get_softc(dev); 596a9e644cdSLuigi Rizzo 597a9e644cdSLuigi Rizzo (void)sc; 598a9e644cdSLuigi Rizzo 599a9e644cdSLuigi Rizzo return (0); 600a9e644cdSLuigi Rizzo } 601a9e644cdSLuigi Rizzo 602a9e644cdSLuigi Rizzo static int 603a9e644cdSLuigi Rizzo ptnet_resume(device_t dev) 604a9e644cdSLuigi Rizzo { 605b6e66be2SVincenzo Maffione struct ptnet_softc *sc = device_get_softc(dev); 606a9e644cdSLuigi Rizzo 607a9e644cdSLuigi Rizzo (void)sc; 608a9e644cdSLuigi Rizzo 609a9e644cdSLuigi Rizzo return (0); 610a9e644cdSLuigi Rizzo } 611a9e644cdSLuigi Rizzo 612a9e644cdSLuigi Rizzo static int 613a9e644cdSLuigi Rizzo ptnet_shutdown(device_t dev) 614a9e644cdSLuigi Rizzo { 615b6e66be2SVincenzo Maffione struct ptnet_softc *sc = device_get_softc(dev); 616b6e66be2SVincenzo Maffione 617b6e66be2SVincenzo Maffione ptnet_device_shutdown(sc); 618b6e66be2SVincenzo Maffione 619b6e66be2SVincenzo Maffione return (0); 620a9e644cdSLuigi Rizzo } 621a9e644cdSLuigi Rizzo 622a9e644cdSLuigi Rizzo static int 623a9e644cdSLuigi Rizzo ptnet_irqs_init(struct ptnet_softc *sc) 624a9e644cdSLuigi Rizzo { 625a9e644cdSLuigi Rizzo int rid = PCIR_BAR(PTNETMAP_MSIX_PCI_BAR); 626a9e644cdSLuigi Rizzo int nvecs = sc->num_rings; 627a9e644cdSLuigi Rizzo device_t dev = sc->dev; 628a9e644cdSLuigi Rizzo int err = ENOSPC; 629a9e644cdSLuigi Rizzo int cpu_cur; 630a9e644cdSLuigi Rizzo int i; 631a9e644cdSLuigi Rizzo 632a9e644cdSLuigi Rizzo if (pci_find_cap(dev, PCIY_MSIX, NULL) != 0) { 633a9e644cdSLuigi Rizzo device_printf(dev, "Could not find MSI-X capability\n"); 634a9e644cdSLuigi Rizzo return (ENXIO); 635a9e644cdSLuigi Rizzo } 636a9e644cdSLuigi Rizzo 637a9e644cdSLuigi Rizzo sc->msix_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 638a9e644cdSLuigi Rizzo &rid, RF_ACTIVE); 639a9e644cdSLuigi Rizzo if (sc->msix_mem == NULL) { 640a9e644cdSLuigi Rizzo device_printf(dev, "Failed to allocate MSIX PCI BAR\n"); 641a9e644cdSLuigi Rizzo return (ENXIO); 642a9e644cdSLuigi Rizzo } 643a9e644cdSLuigi Rizzo 644a9e644cdSLuigi Rizzo if (pci_msix_count(dev) < nvecs) { 645a9e644cdSLuigi Rizzo device_printf(dev, "Not enough MSI-X vectors\n"); 646a9e644cdSLuigi Rizzo goto err_path; 647a9e644cdSLuigi Rizzo } 648a9e644cdSLuigi Rizzo 649a9e644cdSLuigi Rizzo err = pci_alloc_msix(dev, &nvecs); 650a9e644cdSLuigi Rizzo if (err) { 651a9e644cdSLuigi Rizzo device_printf(dev, "Failed to allocate MSI-X vectors\n"); 652a9e644cdSLuigi Rizzo goto err_path; 653a9e644cdSLuigi Rizzo } 654a9e644cdSLuigi Rizzo 655a9e644cdSLuigi Rizzo for (i = 0; i < nvecs; i++) { 656a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 657a9e644cdSLuigi Rizzo 658a9e644cdSLuigi Rizzo rid = i + 1; 659a9e644cdSLuigi Rizzo pq->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 660a9e644cdSLuigi Rizzo RF_ACTIVE); 661a9e644cdSLuigi Rizzo if (pq->irq == NULL) { 662a9e644cdSLuigi Rizzo device_printf(dev, "Failed to allocate interrupt " 663a9e644cdSLuigi Rizzo "for queue #%d\n", i); 664a9e644cdSLuigi Rizzo err = ENOSPC; 665a9e644cdSLuigi Rizzo goto err_path; 666a9e644cdSLuigi Rizzo } 667a9e644cdSLuigi Rizzo } 668a9e644cdSLuigi Rizzo 669a9e644cdSLuigi Rizzo cpu_cur = CPU_FIRST(); 670a9e644cdSLuigi Rizzo for (i = 0; i < nvecs; i++) { 671a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 672a9e644cdSLuigi Rizzo void (*handler)(void *) = ptnet_tx_intr; 673a9e644cdSLuigi Rizzo 674a9e644cdSLuigi Rizzo if (i >= sc->num_tx_rings) { 675a9e644cdSLuigi Rizzo handler = ptnet_rx_intr; 676a9e644cdSLuigi Rizzo } 677a9e644cdSLuigi Rizzo err = bus_setup_intr(dev, pq->irq, INTR_TYPE_NET | INTR_MPSAFE, 678a9e644cdSLuigi Rizzo NULL /* intr_filter */, handler, 679a9e644cdSLuigi Rizzo pq, &pq->cookie); 680a9e644cdSLuigi Rizzo if (err) { 681a9e644cdSLuigi Rizzo device_printf(dev, "Failed to register intr handler " 682a9e644cdSLuigi Rizzo "for queue #%d\n", i); 683a9e644cdSLuigi Rizzo goto err_path; 684a9e644cdSLuigi Rizzo } 685a9e644cdSLuigi Rizzo 686a9e644cdSLuigi Rizzo bus_describe_intr(dev, pq->irq, pq->cookie, "q%d", i); 687a9e644cdSLuigi Rizzo #if 0 688a9e644cdSLuigi Rizzo bus_bind_intr(sc->dev, pq->irq, cpu_cur); 689a9e644cdSLuigi Rizzo #endif 690a9e644cdSLuigi Rizzo cpu_cur = CPU_NEXT(cpu_cur); 691a9e644cdSLuigi Rizzo } 692a9e644cdSLuigi Rizzo 693a9e644cdSLuigi Rizzo device_printf(dev, "Allocated %d MSI-X vectors\n", nvecs); 694a9e644cdSLuigi Rizzo 695a9e644cdSLuigi Rizzo cpu_cur = CPU_FIRST(); 696a9e644cdSLuigi Rizzo for (i = 0; i < nvecs; i++) { 697a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 698a9e644cdSLuigi Rizzo static void (*handler)(void *context, int pending); 699a9e644cdSLuigi Rizzo 700a9e644cdSLuigi Rizzo handler = (i < sc->num_tx_rings) ? ptnet_tx_task : ptnet_rx_task; 701a9e644cdSLuigi Rizzo 702a9e644cdSLuigi Rizzo TASK_INIT(&pq->task, 0, handler, pq); 703a9e644cdSLuigi Rizzo pq->taskq = taskqueue_create_fast("ptnet_queue", M_NOWAIT, 704a9e644cdSLuigi Rizzo taskqueue_thread_enqueue, &pq->taskq); 705a9e644cdSLuigi Rizzo taskqueue_start_threads(&pq->taskq, 1, PI_NET, "%s-pq-%d", 706a9e644cdSLuigi Rizzo device_get_nameunit(sc->dev), cpu_cur); 707a9e644cdSLuigi Rizzo cpu_cur = CPU_NEXT(cpu_cur); 708a9e644cdSLuigi Rizzo } 709a9e644cdSLuigi Rizzo 710a9e644cdSLuigi Rizzo return 0; 711a9e644cdSLuigi Rizzo err_path: 712a9e644cdSLuigi Rizzo ptnet_irqs_fini(sc); 713a9e644cdSLuigi Rizzo return err; 714a9e644cdSLuigi Rizzo } 715a9e644cdSLuigi Rizzo 716a9e644cdSLuigi Rizzo static void 717a9e644cdSLuigi Rizzo ptnet_irqs_fini(struct ptnet_softc *sc) 718a9e644cdSLuigi Rizzo { 719a9e644cdSLuigi Rizzo device_t dev = sc->dev; 720a9e644cdSLuigi Rizzo int i; 721a9e644cdSLuigi Rizzo 722a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 723a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 724a9e644cdSLuigi Rizzo 725a9e644cdSLuigi Rizzo if (pq->taskq) { 726a9e644cdSLuigi Rizzo taskqueue_free(pq->taskq); 727a9e644cdSLuigi Rizzo pq->taskq = NULL; 728a9e644cdSLuigi Rizzo } 729a9e644cdSLuigi Rizzo 730a9e644cdSLuigi Rizzo if (pq->cookie) { 731a9e644cdSLuigi Rizzo bus_teardown_intr(dev, pq->irq, pq->cookie); 732a9e644cdSLuigi Rizzo pq->cookie = NULL; 733a9e644cdSLuigi Rizzo } 734a9e644cdSLuigi Rizzo 735a9e644cdSLuigi Rizzo if (pq->irq) { 736a9e644cdSLuigi Rizzo bus_release_resource(dev, SYS_RES_IRQ, i + 1, pq->irq); 737a9e644cdSLuigi Rizzo pq->irq = NULL; 738a9e644cdSLuigi Rizzo } 739a9e644cdSLuigi Rizzo } 740a9e644cdSLuigi Rizzo 741a9e644cdSLuigi Rizzo if (sc->msix_mem) { 742a9e644cdSLuigi Rizzo pci_release_msi(dev); 743a9e644cdSLuigi Rizzo 744a9e644cdSLuigi Rizzo bus_release_resource(dev, SYS_RES_MEMORY, 745a9e644cdSLuigi Rizzo PCIR_BAR(PTNETMAP_MSIX_PCI_BAR), 746a9e644cdSLuigi Rizzo sc->msix_mem); 747a9e644cdSLuigi Rizzo sc->msix_mem = NULL; 748a9e644cdSLuigi Rizzo } 749a9e644cdSLuigi Rizzo } 750a9e644cdSLuigi Rizzo 751a9e644cdSLuigi Rizzo static void 752a9e644cdSLuigi Rizzo ptnet_init(void *opaque) 753a9e644cdSLuigi Rizzo { 754a9e644cdSLuigi Rizzo struct ptnet_softc *sc = opaque; 755a9e644cdSLuigi Rizzo 756a9e644cdSLuigi Rizzo PTNET_CORE_LOCK(sc); 757a9e644cdSLuigi Rizzo ptnet_init_locked(sc); 758a9e644cdSLuigi Rizzo PTNET_CORE_UNLOCK(sc); 759a9e644cdSLuigi Rizzo } 760a9e644cdSLuigi Rizzo 761a9e644cdSLuigi Rizzo static int 762a9e644cdSLuigi Rizzo ptnet_ioctl(if_t ifp, u_long cmd, caddr_t data) 763a9e644cdSLuigi Rizzo { 764a9e644cdSLuigi Rizzo struct ptnet_softc *sc = if_getsoftc(ifp); 765a9e644cdSLuigi Rizzo device_t dev = sc->dev; 766a9e644cdSLuigi Rizzo struct ifreq *ifr = (struct ifreq *)data; 76724a7d6d3SMatt Macy int mask __unused, err = 0; 768a9e644cdSLuigi Rizzo 769a9e644cdSLuigi Rizzo switch (cmd) { 770a9e644cdSLuigi Rizzo case SIOCSIFFLAGS: 771a9e644cdSLuigi Rizzo device_printf(dev, "SIOCSIFFLAGS %x\n", ifp->if_flags); 772a9e644cdSLuigi Rizzo PTNET_CORE_LOCK(sc); 773a9e644cdSLuigi Rizzo if (ifp->if_flags & IFF_UP) { 774a9e644cdSLuigi Rizzo /* Network stack wants the iff to be up. */ 775a9e644cdSLuigi Rizzo err = ptnet_init_locked(sc); 776a9e644cdSLuigi Rizzo } else { 777a9e644cdSLuigi Rizzo /* Network stack wants the iff to be down. */ 778a9e644cdSLuigi Rizzo err = ptnet_stop(sc); 779a9e644cdSLuigi Rizzo } 780a9e644cdSLuigi Rizzo /* We don't need to do nothing to support IFF_PROMISC, 781a9e644cdSLuigi Rizzo * since that is managed by the backend port. */ 782a9e644cdSLuigi Rizzo PTNET_CORE_UNLOCK(sc); 783a9e644cdSLuigi Rizzo break; 784a9e644cdSLuigi Rizzo 785a9e644cdSLuigi Rizzo case SIOCSIFCAP: 786a9e644cdSLuigi Rizzo device_printf(dev, "SIOCSIFCAP %x %x\n", 787a9e644cdSLuigi Rizzo ifr->ifr_reqcap, ifp->if_capenable); 788a9e644cdSLuigi Rizzo mask = ifr->ifr_reqcap ^ ifp->if_capenable; 789a9e644cdSLuigi Rizzo #ifdef DEVICE_POLLING 790a9e644cdSLuigi Rizzo if (mask & IFCAP_POLLING) { 791a9e644cdSLuigi Rizzo struct ptnet_queue *pq; 792a9e644cdSLuigi Rizzo int i; 793a9e644cdSLuigi Rizzo 794a9e644cdSLuigi Rizzo if (ifr->ifr_reqcap & IFCAP_POLLING) { 795a9e644cdSLuigi Rizzo err = ether_poll_register(ptnet_poll, ifp); 796a9e644cdSLuigi Rizzo if (err) { 797a9e644cdSLuigi Rizzo break; 798a9e644cdSLuigi Rizzo } 799a9e644cdSLuigi Rizzo /* Stop queues and sync with taskqueues. */ 800a9e644cdSLuigi Rizzo ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 801a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 802a9e644cdSLuigi Rizzo pq = sc-> queues + i; 803a9e644cdSLuigi Rizzo /* Make sure the worker sees the 804a9e644cdSLuigi Rizzo * IFF_DRV_RUNNING down. */ 805a9e644cdSLuigi Rizzo PTNET_Q_LOCK(pq); 806b6e66be2SVincenzo Maffione pq->atok->appl_need_kick = 0; 807a9e644cdSLuigi Rizzo PTNET_Q_UNLOCK(pq); 808a9e644cdSLuigi Rizzo /* Wait for rescheduling to finish. */ 809a9e644cdSLuigi Rizzo if (pq->taskq) { 810a9e644cdSLuigi Rizzo taskqueue_drain(pq->taskq, 811a9e644cdSLuigi Rizzo &pq->task); 812a9e644cdSLuigi Rizzo } 813a9e644cdSLuigi Rizzo } 814a9e644cdSLuigi Rizzo ifp->if_drv_flags |= IFF_DRV_RUNNING; 815a9e644cdSLuigi Rizzo } else { 816a9e644cdSLuigi Rizzo err = ether_poll_deregister(ifp); 817a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 818a9e644cdSLuigi Rizzo pq = sc-> queues + i; 819a9e644cdSLuigi Rizzo PTNET_Q_LOCK(pq); 820b6e66be2SVincenzo Maffione pq->atok->appl_need_kick = 1; 821a9e644cdSLuigi Rizzo PTNET_Q_UNLOCK(pq); 822a9e644cdSLuigi Rizzo } 823a9e644cdSLuigi Rizzo } 824a9e644cdSLuigi Rizzo } 825a9e644cdSLuigi Rizzo #endif /* DEVICE_POLLING */ 826a9e644cdSLuigi Rizzo ifp->if_capenable = ifr->ifr_reqcap; 827a9e644cdSLuigi Rizzo break; 828a9e644cdSLuigi Rizzo 829a9e644cdSLuigi Rizzo case SIOCSIFMTU: 830a9e644cdSLuigi Rizzo /* We support any reasonable MTU. */ 831a9e644cdSLuigi Rizzo if (ifr->ifr_mtu < ETHERMIN || 832a9e644cdSLuigi Rizzo ifr->ifr_mtu > PTNET_MAX_PKT_SIZE) { 833a9e644cdSLuigi Rizzo err = EINVAL; 834a9e644cdSLuigi Rizzo } else { 835a9e644cdSLuigi Rizzo PTNET_CORE_LOCK(sc); 836a9e644cdSLuigi Rizzo ifp->if_mtu = ifr->ifr_mtu; 837a9e644cdSLuigi Rizzo PTNET_CORE_UNLOCK(sc); 838a9e644cdSLuigi Rizzo } 839a9e644cdSLuigi Rizzo break; 840a9e644cdSLuigi Rizzo 841a9e644cdSLuigi Rizzo case SIOCSIFMEDIA: 842a9e644cdSLuigi Rizzo case SIOCGIFMEDIA: 843a9e644cdSLuigi Rizzo err = ifmedia_ioctl(ifp, ifr, &sc->media, cmd); 844a9e644cdSLuigi Rizzo break; 845a9e644cdSLuigi Rizzo 846a9e644cdSLuigi Rizzo default: 847a9e644cdSLuigi Rizzo err = ether_ioctl(ifp, cmd, data); 848a9e644cdSLuigi Rizzo break; 849a9e644cdSLuigi Rizzo } 850a9e644cdSLuigi Rizzo 851a9e644cdSLuigi Rizzo return err; 852a9e644cdSLuigi Rizzo } 853a9e644cdSLuigi Rizzo 854a9e644cdSLuigi Rizzo static int 855a9e644cdSLuigi Rizzo ptnet_init_locked(struct ptnet_softc *sc) 856a9e644cdSLuigi Rizzo { 857a9e644cdSLuigi Rizzo if_t ifp = sc->ifp; 858a9e644cdSLuigi Rizzo struct netmap_adapter *na_dr = &sc->ptna->dr.up; 859a9e644cdSLuigi Rizzo struct netmap_adapter *na_nm = &sc->ptna->hwup.up; 860a9e644cdSLuigi Rizzo unsigned int nm_buf_size; 861a9e644cdSLuigi Rizzo int ret; 862a9e644cdSLuigi Rizzo 863a9e644cdSLuigi Rizzo if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 864a9e644cdSLuigi Rizzo return 0; /* nothing to do */ 865a9e644cdSLuigi Rizzo } 866a9e644cdSLuigi Rizzo 867a9e644cdSLuigi Rizzo device_printf(sc->dev, "%s\n", __func__); 868a9e644cdSLuigi Rizzo 869a9e644cdSLuigi Rizzo /* Translate offload capabilities according to if_capenable. */ 870a9e644cdSLuigi Rizzo ifp->if_hwassist = 0; 871a9e644cdSLuigi Rizzo if (ifp->if_capenable & IFCAP_TXCSUM) 872a9e644cdSLuigi Rizzo ifp->if_hwassist |= PTNET_CSUM_OFFLOAD; 873a9e644cdSLuigi Rizzo if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) 874a9e644cdSLuigi Rizzo ifp->if_hwassist |= PTNET_CSUM_OFFLOAD_IPV6; 875a9e644cdSLuigi Rizzo if (ifp->if_capenable & IFCAP_TSO4) 876a9e644cdSLuigi Rizzo ifp->if_hwassist |= CSUM_IP_TSO; 877a9e644cdSLuigi Rizzo if (ifp->if_capenable & IFCAP_TSO6) 878a9e644cdSLuigi Rizzo ifp->if_hwassist |= CSUM_IP6_TSO; 879a9e644cdSLuigi Rizzo 880a9e644cdSLuigi Rizzo /* 881a9e644cdSLuigi Rizzo * Prepare the interface for netmap mode access. 882a9e644cdSLuigi Rizzo */ 883a9e644cdSLuigi Rizzo netmap_update_config(na_dr); 884a9e644cdSLuigi Rizzo 885a9e644cdSLuigi Rizzo ret = netmap_mem_finalize(na_dr->nm_mem, na_dr); 886a9e644cdSLuigi Rizzo if (ret) { 887a9e644cdSLuigi Rizzo device_printf(sc->dev, "netmap_mem_finalize() failed\n"); 888a9e644cdSLuigi Rizzo return ret; 889a9e644cdSLuigi Rizzo } 890a9e644cdSLuigi Rizzo 891b6e66be2SVincenzo Maffione if (sc->ptna->backend_users == 0) { 892a9e644cdSLuigi Rizzo ret = ptnet_nm_krings_create(na_nm); 893a9e644cdSLuigi Rizzo if (ret) { 894a9e644cdSLuigi Rizzo device_printf(sc->dev, "ptnet_nm_krings_create() " 895a9e644cdSLuigi Rizzo "failed\n"); 896a9e644cdSLuigi Rizzo goto err_mem_finalize; 897a9e644cdSLuigi Rizzo } 898a9e644cdSLuigi Rizzo 899a9e644cdSLuigi Rizzo ret = netmap_mem_rings_create(na_dr); 900a9e644cdSLuigi Rizzo if (ret) { 901a9e644cdSLuigi Rizzo device_printf(sc->dev, "netmap_mem_rings_create() " 902a9e644cdSLuigi Rizzo "failed\n"); 903a9e644cdSLuigi Rizzo goto err_rings_create; 904a9e644cdSLuigi Rizzo } 905a9e644cdSLuigi Rizzo 906a9e644cdSLuigi Rizzo ret = netmap_mem_get_lut(na_dr->nm_mem, &na_dr->na_lut); 907a9e644cdSLuigi Rizzo if (ret) { 908a9e644cdSLuigi Rizzo device_printf(sc->dev, "netmap_mem_get_lut() " 909a9e644cdSLuigi Rizzo "failed\n"); 910a9e644cdSLuigi Rizzo goto err_get_lut; 911a9e644cdSLuigi Rizzo } 912a9e644cdSLuigi Rizzo } 913a9e644cdSLuigi Rizzo 914a9e644cdSLuigi Rizzo ret = ptnet_nm_register(na_dr, 1 /* on */); 915a9e644cdSLuigi Rizzo if (ret) { 916a9e644cdSLuigi Rizzo goto err_register; 917a9e644cdSLuigi Rizzo } 918a9e644cdSLuigi Rizzo 919a9e644cdSLuigi Rizzo nm_buf_size = NETMAP_BUF_SIZE(na_dr); 920a9e644cdSLuigi Rizzo 921a9e644cdSLuigi Rizzo KASSERT(nm_buf_size > 0, ("Invalid netmap buffer size")); 922a9e644cdSLuigi Rizzo sc->min_tx_space = PTNET_MAX_PKT_SIZE / nm_buf_size + 2; 923a9e644cdSLuigi Rizzo device_printf(sc->dev, "%s: min_tx_space = %u\n", __func__, 924a9e644cdSLuigi Rizzo sc->min_tx_space); 925a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS 926a9e644cdSLuigi Rizzo callout_reset(&sc->tick, hz, ptnet_tick, sc); 927a9e644cdSLuigi Rizzo #endif 928a9e644cdSLuigi Rizzo 929a9e644cdSLuigi Rizzo ifp->if_drv_flags |= IFF_DRV_RUNNING; 930a9e644cdSLuigi Rizzo 931a9e644cdSLuigi Rizzo return 0; 932a9e644cdSLuigi Rizzo 933a9e644cdSLuigi Rizzo err_register: 934a9e644cdSLuigi Rizzo memset(&na_dr->na_lut, 0, sizeof(na_dr->na_lut)); 935a9e644cdSLuigi Rizzo err_get_lut: 936a9e644cdSLuigi Rizzo netmap_mem_rings_delete(na_dr); 937a9e644cdSLuigi Rizzo err_rings_create: 938a9e644cdSLuigi Rizzo ptnet_nm_krings_delete(na_nm); 939a9e644cdSLuigi Rizzo err_mem_finalize: 940a9e644cdSLuigi Rizzo netmap_mem_deref(na_dr->nm_mem, na_dr); 941a9e644cdSLuigi Rizzo 942a9e644cdSLuigi Rizzo return ret; 943a9e644cdSLuigi Rizzo } 944a9e644cdSLuigi Rizzo 945a9e644cdSLuigi Rizzo /* To be called under core lock. */ 946a9e644cdSLuigi Rizzo static int 947a9e644cdSLuigi Rizzo ptnet_stop(struct ptnet_softc *sc) 948a9e644cdSLuigi Rizzo { 949a9e644cdSLuigi Rizzo if_t ifp = sc->ifp; 950a9e644cdSLuigi Rizzo struct netmap_adapter *na_dr = &sc->ptna->dr.up; 951a9e644cdSLuigi Rizzo struct netmap_adapter *na_nm = &sc->ptna->hwup.up; 952a9e644cdSLuigi Rizzo int i; 953a9e644cdSLuigi Rizzo 954a9e644cdSLuigi Rizzo device_printf(sc->dev, "%s\n", __func__); 955a9e644cdSLuigi Rizzo 956a9e644cdSLuigi Rizzo if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 957a9e644cdSLuigi Rizzo return 0; /* nothing to do */ 958a9e644cdSLuigi Rizzo } 959a9e644cdSLuigi Rizzo 960a9e644cdSLuigi Rizzo /* Clear the driver-ready flag, and synchronize with all the queues, 961a9e644cdSLuigi Rizzo * so that after this loop we are sure nobody is working anymore with 962a9e644cdSLuigi Rizzo * the device. This scheme is taken from the vtnet driver. */ 963a9e644cdSLuigi Rizzo ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 964a9e644cdSLuigi Rizzo callout_stop(&sc->tick); 965a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 966a9e644cdSLuigi Rizzo PTNET_Q_LOCK(sc->queues + i); 967a9e644cdSLuigi Rizzo PTNET_Q_UNLOCK(sc->queues + i); 968a9e644cdSLuigi Rizzo } 969a9e644cdSLuigi Rizzo 970a9e644cdSLuigi Rizzo ptnet_nm_register(na_dr, 0 /* off */); 971a9e644cdSLuigi Rizzo 972b6e66be2SVincenzo Maffione if (sc->ptna->backend_users == 0) { 973a9e644cdSLuigi Rizzo netmap_mem_rings_delete(na_dr); 974a9e644cdSLuigi Rizzo ptnet_nm_krings_delete(na_nm); 975a9e644cdSLuigi Rizzo } 976a9e644cdSLuigi Rizzo netmap_mem_deref(na_dr->nm_mem, na_dr); 977a9e644cdSLuigi Rizzo 978a9e644cdSLuigi Rizzo return 0; 979a9e644cdSLuigi Rizzo } 980a9e644cdSLuigi Rizzo 981a9e644cdSLuigi Rizzo static void 982a9e644cdSLuigi Rizzo ptnet_qflush(if_t ifp) 983a9e644cdSLuigi Rizzo { 984a9e644cdSLuigi Rizzo struct ptnet_softc *sc = if_getsoftc(ifp); 985a9e644cdSLuigi Rizzo int i; 986a9e644cdSLuigi Rizzo 987a9e644cdSLuigi Rizzo /* Flush all the bufrings and do the interface flush. */ 988a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 989a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 990a9e644cdSLuigi Rizzo struct mbuf *m; 991a9e644cdSLuigi Rizzo 992a9e644cdSLuigi Rizzo PTNET_Q_LOCK(pq); 993a9e644cdSLuigi Rizzo if (pq->bufring) { 994a9e644cdSLuigi Rizzo while ((m = buf_ring_dequeue_sc(pq->bufring))) { 995a9e644cdSLuigi Rizzo m_freem(m); 996a9e644cdSLuigi Rizzo } 997a9e644cdSLuigi Rizzo } 998a9e644cdSLuigi Rizzo PTNET_Q_UNLOCK(pq); 999a9e644cdSLuigi Rizzo } 1000a9e644cdSLuigi Rizzo 1001a9e644cdSLuigi Rizzo if_qflush(ifp); 1002a9e644cdSLuigi Rizzo } 1003a9e644cdSLuigi Rizzo 1004a9e644cdSLuigi Rizzo static int 1005a9e644cdSLuigi Rizzo ptnet_media_change(if_t ifp) 1006a9e644cdSLuigi Rizzo { 1007a9e644cdSLuigi Rizzo struct ptnet_softc *sc = if_getsoftc(ifp); 1008a9e644cdSLuigi Rizzo struct ifmedia *ifm = &sc->media; 1009a9e644cdSLuigi Rizzo 1010a9e644cdSLuigi Rizzo if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) { 1011a9e644cdSLuigi Rizzo return EINVAL; 1012a9e644cdSLuigi Rizzo } 1013a9e644cdSLuigi Rizzo 1014a9e644cdSLuigi Rizzo return 0; 1015a9e644cdSLuigi Rizzo } 1016a9e644cdSLuigi Rizzo 1017a9e644cdSLuigi Rizzo #if __FreeBSD_version >= 1100000 1018a9e644cdSLuigi Rizzo static uint64_t 1019a9e644cdSLuigi Rizzo ptnet_get_counter(if_t ifp, ift_counter cnt) 1020a9e644cdSLuigi Rizzo { 1021a9e644cdSLuigi Rizzo struct ptnet_softc *sc = if_getsoftc(ifp); 1022a9e644cdSLuigi Rizzo struct ptnet_queue_stats stats[2]; 1023a9e644cdSLuigi Rizzo int i; 1024a9e644cdSLuigi Rizzo 1025a9e644cdSLuigi Rizzo /* Accumulate statistics over the queues. */ 1026a9e644cdSLuigi Rizzo memset(stats, 0, sizeof(stats)); 1027a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 1028a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 1029a9e644cdSLuigi Rizzo int idx = (i < sc->num_tx_rings) ? 0 : 1; 1030a9e644cdSLuigi Rizzo 1031a9e644cdSLuigi Rizzo stats[idx].packets += pq->stats.packets; 1032a9e644cdSLuigi Rizzo stats[idx].bytes += pq->stats.bytes; 1033a9e644cdSLuigi Rizzo stats[idx].errors += pq->stats.errors; 1034a9e644cdSLuigi Rizzo stats[idx].iqdrops += pq->stats.iqdrops; 1035a9e644cdSLuigi Rizzo stats[idx].mcasts += pq->stats.mcasts; 1036a9e644cdSLuigi Rizzo } 1037a9e644cdSLuigi Rizzo 1038a9e644cdSLuigi Rizzo switch (cnt) { 1039a9e644cdSLuigi Rizzo case IFCOUNTER_IPACKETS: 1040a9e644cdSLuigi Rizzo return (stats[1].packets); 1041a9e644cdSLuigi Rizzo case IFCOUNTER_IQDROPS: 1042a9e644cdSLuigi Rizzo return (stats[1].iqdrops); 1043a9e644cdSLuigi Rizzo case IFCOUNTER_IERRORS: 1044a9e644cdSLuigi Rizzo return (stats[1].errors); 1045a9e644cdSLuigi Rizzo case IFCOUNTER_OPACKETS: 1046a9e644cdSLuigi Rizzo return (stats[0].packets); 1047a9e644cdSLuigi Rizzo case IFCOUNTER_OBYTES: 1048a9e644cdSLuigi Rizzo return (stats[0].bytes); 1049a9e644cdSLuigi Rizzo case IFCOUNTER_OMCASTS: 1050a9e644cdSLuigi Rizzo return (stats[0].mcasts); 1051a9e644cdSLuigi Rizzo default: 1052a9e644cdSLuigi Rizzo return (if_get_counter_default(ifp, cnt)); 1053a9e644cdSLuigi Rizzo } 1054a9e644cdSLuigi Rizzo } 1055a9e644cdSLuigi Rizzo #endif 1056a9e644cdSLuigi Rizzo 1057a9e644cdSLuigi Rizzo 1058a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS 1059a9e644cdSLuigi Rizzo /* Called under core lock. */ 1060a9e644cdSLuigi Rizzo static void 1061a9e644cdSLuigi Rizzo ptnet_tick(void *opaque) 1062a9e644cdSLuigi Rizzo { 1063a9e644cdSLuigi Rizzo struct ptnet_softc *sc = opaque; 1064a9e644cdSLuigi Rizzo int i; 1065a9e644cdSLuigi Rizzo 1066a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 1067a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 1068a9e644cdSLuigi Rizzo struct ptnet_queue_stats cur = pq->stats; 1069a9e644cdSLuigi Rizzo struct timeval now; 1070a9e644cdSLuigi Rizzo unsigned int delta; 1071a9e644cdSLuigi Rizzo 1072a9e644cdSLuigi Rizzo microtime(&now); 1073a9e644cdSLuigi Rizzo delta = now.tv_usec - sc->last_ts.tv_usec + 1074a9e644cdSLuigi Rizzo (now.tv_sec - sc->last_ts.tv_sec) * 1000000; 1075a9e644cdSLuigi Rizzo delta /= 1000; /* in milliseconds */ 1076a9e644cdSLuigi Rizzo 1077a9e644cdSLuigi Rizzo if (delta == 0) 1078a9e644cdSLuigi Rizzo continue; 1079a9e644cdSLuigi Rizzo 1080a9e644cdSLuigi Rizzo device_printf(sc->dev, "#%d[%u ms]:pkts %lu, kicks %lu, " 1081a9e644cdSLuigi Rizzo "intr %lu\n", i, delta, 1082a9e644cdSLuigi Rizzo (cur.packets - pq->last_stats.packets), 1083a9e644cdSLuigi Rizzo (cur.kicks - pq->last_stats.kicks), 1084a9e644cdSLuigi Rizzo (cur.intrs - pq->last_stats.intrs)); 1085a9e644cdSLuigi Rizzo pq->last_stats = cur; 1086a9e644cdSLuigi Rizzo } 1087a9e644cdSLuigi Rizzo microtime(&sc->last_ts); 1088a9e644cdSLuigi Rizzo callout_schedule(&sc->tick, hz); 1089a9e644cdSLuigi Rizzo } 1090a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */ 1091a9e644cdSLuigi Rizzo 1092a9e644cdSLuigi Rizzo static void 1093a9e644cdSLuigi Rizzo ptnet_media_status(if_t ifp, struct ifmediareq *ifmr) 1094a9e644cdSLuigi Rizzo { 1095a9e644cdSLuigi Rizzo /* We are always active, as the backend netmap port is 1096a9e644cdSLuigi Rizzo * always open in netmap mode. */ 1097a9e644cdSLuigi Rizzo ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE; 1098a9e644cdSLuigi Rizzo ifmr->ifm_active = IFM_ETHER | IFM_10G_T | IFM_FDX; 1099a9e644cdSLuigi Rizzo } 1100a9e644cdSLuigi Rizzo 1101a9e644cdSLuigi Rizzo static uint32_t 1102b6e66be2SVincenzo Maffione ptnet_nm_ptctl(struct ptnet_softc *sc, uint32_t cmd) 1103a9e644cdSLuigi Rizzo { 1104844a6f0cSLuigi Rizzo /* 1105844a6f0cSLuigi Rizzo * Write a command and read back error status, 1106844a6f0cSLuigi Rizzo * with zero meaning success. 1107844a6f0cSLuigi Rizzo */ 1108a9e644cdSLuigi Rizzo bus_write_4(sc->iomem, PTNET_IO_PTCTL, cmd); 1109844a6f0cSLuigi Rizzo return bus_read_4(sc->iomem, PTNET_IO_PTCTL); 1110a9e644cdSLuigi Rizzo } 1111a9e644cdSLuigi Rizzo 1112a9e644cdSLuigi Rizzo static int 11132ff91c17SVincenzo Maffione ptnet_nm_config(struct netmap_adapter *na, struct nm_config_info *info) 1114a9e644cdSLuigi Rizzo { 1115a9e644cdSLuigi Rizzo struct ptnet_softc *sc = if_getsoftc(na->ifp); 1116a9e644cdSLuigi Rizzo 11172ff91c17SVincenzo Maffione info->num_tx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_RINGS); 11182ff91c17SVincenzo Maffione info->num_rx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_RINGS); 11192ff91c17SVincenzo Maffione info->num_tx_descs = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_SLOTS); 11202ff91c17SVincenzo Maffione info->num_rx_descs = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_SLOTS); 11212ff91c17SVincenzo Maffione info->rx_buf_maxsize = NETMAP_BUF_SIZE(na); 1122a9e644cdSLuigi Rizzo 11232ff91c17SVincenzo Maffione device_printf(sc->dev, "txr %u, rxr %u, txd %u, rxd %u, rxbufsz %u\n", 11242ff91c17SVincenzo Maffione info->num_tx_rings, info->num_rx_rings, 11252ff91c17SVincenzo Maffione info->num_tx_descs, info->num_rx_descs, 11262ff91c17SVincenzo Maffione info->rx_buf_maxsize); 1127a9e644cdSLuigi Rizzo 1128a9e644cdSLuigi Rizzo return 0; 1129a9e644cdSLuigi Rizzo } 1130a9e644cdSLuigi Rizzo 1131a9e644cdSLuigi Rizzo static void 1132a9e644cdSLuigi Rizzo ptnet_sync_from_csb(struct ptnet_softc *sc, struct netmap_adapter *na) 1133a9e644cdSLuigi Rizzo { 1134a9e644cdSLuigi Rizzo int i; 1135a9e644cdSLuigi Rizzo 1136a9e644cdSLuigi Rizzo /* Sync krings from the host, reading from 1137a9e644cdSLuigi Rizzo * CSB. */ 1138a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 1139b6e66be2SVincenzo Maffione struct nm_csb_atok *atok = sc->queues[i].atok; 1140b6e66be2SVincenzo Maffione struct nm_csb_ktoa *ktoa = sc->queues[i].ktoa; 1141a9e644cdSLuigi Rizzo struct netmap_kring *kring; 1142a9e644cdSLuigi Rizzo 1143a9e644cdSLuigi Rizzo if (i < na->num_tx_rings) { 11442ff91c17SVincenzo Maffione kring = na->tx_rings[i]; 1145a9e644cdSLuigi Rizzo } else { 11462ff91c17SVincenzo Maffione kring = na->rx_rings[i - na->num_tx_rings]; 1147a9e644cdSLuigi Rizzo } 1148b6e66be2SVincenzo Maffione kring->rhead = kring->ring->head = atok->head; 1149b6e66be2SVincenzo Maffione kring->rcur = kring->ring->cur = atok->cur; 1150b6e66be2SVincenzo Maffione kring->nr_hwcur = ktoa->hwcur; 1151a9e644cdSLuigi Rizzo kring->nr_hwtail = kring->rtail = 1152b6e66be2SVincenzo Maffione kring->ring->tail = ktoa->hwtail; 1153a9e644cdSLuigi Rizzo 115475f4f3edSVincenzo Maffione nm_prdis("%d,%d: csb {hc %u h %u c %u ht %u}", t, i, 1155b6e66be2SVincenzo Maffione ktoa->hwcur, atok->head, atok->cur, 1156b6e66be2SVincenzo Maffione ktoa->hwtail); 115775f4f3edSVincenzo Maffione nm_prdis("%d,%d: kring {hc %u rh %u rc %u h %u c %u ht %u rt %u t %u}", 1158a9e644cdSLuigi Rizzo t, i, kring->nr_hwcur, kring->rhead, kring->rcur, 1159a9e644cdSLuigi Rizzo kring->ring->head, kring->ring->cur, kring->nr_hwtail, 1160a9e644cdSLuigi Rizzo kring->rtail, kring->ring->tail); 1161a9e644cdSLuigi Rizzo } 1162a9e644cdSLuigi Rizzo } 1163a9e644cdSLuigi Rizzo 1164a9e644cdSLuigi Rizzo static void 1165a9e644cdSLuigi Rizzo ptnet_update_vnet_hdr(struct ptnet_softc *sc) 1166a9e644cdSLuigi Rizzo { 1167a2a74091SLuigi Rizzo unsigned int wanted_hdr_len = ptnet_vnet_hdr ? PTNET_HDR_SIZE : 0; 1168a2a74091SLuigi Rizzo 1169a2a74091SLuigi Rizzo bus_write_4(sc->iomem, PTNET_IO_VNET_HDR_LEN, wanted_hdr_len); 1170a2a74091SLuigi Rizzo sc->vnet_hdr_len = bus_read_4(sc->iomem, PTNET_IO_VNET_HDR_LEN); 1171a9e644cdSLuigi Rizzo sc->ptna->hwup.up.virt_hdr_len = sc->vnet_hdr_len; 1172a9e644cdSLuigi Rizzo } 1173a9e644cdSLuigi Rizzo 1174a9e644cdSLuigi Rizzo static int 1175a9e644cdSLuigi Rizzo ptnet_nm_register(struct netmap_adapter *na, int onoff) 1176a9e644cdSLuigi Rizzo { 1177a9e644cdSLuigi Rizzo /* device-specific */ 1178a9e644cdSLuigi Rizzo if_t ifp = na->ifp; 1179a9e644cdSLuigi Rizzo struct ptnet_softc *sc = if_getsoftc(ifp); 1180a9e644cdSLuigi Rizzo int native = (na == &sc->ptna->hwup.up); 1181a9e644cdSLuigi Rizzo struct ptnet_queue *pq; 1182a9e644cdSLuigi Rizzo int ret = 0; 1183a9e644cdSLuigi Rizzo int i; 1184a9e644cdSLuigi Rizzo 1185a9e644cdSLuigi Rizzo if (!onoff) { 1186b6e66be2SVincenzo Maffione sc->ptna->backend_users--; 1187a9e644cdSLuigi Rizzo } 1188a9e644cdSLuigi Rizzo 1189a9e644cdSLuigi Rizzo /* If this is the last netmap client, guest interrupt enable flags may 1190a9e644cdSLuigi Rizzo * be in arbitrary state. Since these flags are going to be used also 1191a9e644cdSLuigi Rizzo * by the netdevice driver, we have to make sure to start with 1192a9e644cdSLuigi Rizzo * notifications enabled. Also, schedule NAPI to flush pending packets 1193a9e644cdSLuigi Rizzo * in the RX rings, since we will not receive further interrupts 1194a9e644cdSLuigi Rizzo * until these will be processed. */ 1195a9e644cdSLuigi Rizzo if (native && !onoff && na->active_fds == 0) { 119675f4f3edSVincenzo Maffione nm_prinf("Exit netmap mode, re-enable interrupts"); 1197a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 1198a9e644cdSLuigi Rizzo pq = sc->queues + i; 1199b6e66be2SVincenzo Maffione pq->atok->appl_need_kick = 1; 1200a9e644cdSLuigi Rizzo } 1201a9e644cdSLuigi Rizzo } 1202a9e644cdSLuigi Rizzo 1203a9e644cdSLuigi Rizzo if (onoff) { 1204b6e66be2SVincenzo Maffione if (sc->ptna->backend_users == 0) { 1205a9e644cdSLuigi Rizzo /* Initialize notification enable fields in the CSB. */ 1206a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 1207a9e644cdSLuigi Rizzo pq = sc->queues + i; 1208b6e66be2SVincenzo Maffione pq->ktoa->kern_need_kick = 1; 1209b6e66be2SVincenzo Maffione pq->atok->appl_need_kick = 1210a9e644cdSLuigi Rizzo (!(ifp->if_capenable & IFCAP_POLLING) 1211a9e644cdSLuigi Rizzo && i >= sc->num_tx_rings); 1212a9e644cdSLuigi Rizzo } 1213a9e644cdSLuigi Rizzo 1214a9e644cdSLuigi Rizzo /* Set the virtio-net header length. */ 1215a9e644cdSLuigi Rizzo ptnet_update_vnet_hdr(sc); 1216a9e644cdSLuigi Rizzo 1217a9e644cdSLuigi Rizzo /* Make sure the host adapter passed through is ready 1218a9e644cdSLuigi Rizzo * for txsync/rxsync. */ 1219b6e66be2SVincenzo Maffione ret = ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_CREATE); 1220a9e644cdSLuigi Rizzo if (ret) { 1221a9e644cdSLuigi Rizzo return ret; 1222a9e644cdSLuigi Rizzo } 1223a9e644cdSLuigi Rizzo 1224b6e66be2SVincenzo Maffione /* Align the guest krings and rings to the state stored 1225b6e66be2SVincenzo Maffione * in the CSB. */ 1226a9e644cdSLuigi Rizzo ptnet_sync_from_csb(sc, na); 1227a9e644cdSLuigi Rizzo } 1228a9e644cdSLuigi Rizzo 1229a9e644cdSLuigi Rizzo /* If not native, don't call nm_set_native_flags, since we don't want 1230a9e644cdSLuigi Rizzo * to replace if_transmit method, nor set NAF_NETMAP_ON */ 1231a9e644cdSLuigi Rizzo if (native) { 123275f4f3edSVincenzo Maffione netmap_krings_mode_commit(na, onoff); 1233a9e644cdSLuigi Rizzo nm_set_native_flags(na); 1234a9e644cdSLuigi Rizzo } 1235a9e644cdSLuigi Rizzo 1236a9e644cdSLuigi Rizzo } else { 1237a9e644cdSLuigi Rizzo if (native) { 1238a9e644cdSLuigi Rizzo nm_clear_native_flags(na); 123975f4f3edSVincenzo Maffione netmap_krings_mode_commit(na, onoff); 1240a9e644cdSLuigi Rizzo } 1241a9e644cdSLuigi Rizzo 1242b6e66be2SVincenzo Maffione if (sc->ptna->backend_users == 0) { 1243b6e66be2SVincenzo Maffione ret = ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_DELETE); 1244a9e644cdSLuigi Rizzo } 1245a9e644cdSLuigi Rizzo } 1246a9e644cdSLuigi Rizzo 1247a9e644cdSLuigi Rizzo if (onoff) { 1248b6e66be2SVincenzo Maffione sc->ptna->backend_users++; 1249a9e644cdSLuigi Rizzo } 1250a9e644cdSLuigi Rizzo 1251a9e644cdSLuigi Rizzo return ret; 1252a9e644cdSLuigi Rizzo } 1253a9e644cdSLuigi Rizzo 1254a9e644cdSLuigi Rizzo static int 1255a9e644cdSLuigi Rizzo ptnet_nm_txsync(struct netmap_kring *kring, int flags) 1256a9e644cdSLuigi Rizzo { 1257a9e644cdSLuigi Rizzo struct ptnet_softc *sc = if_getsoftc(kring->na->ifp); 1258a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + kring->ring_id; 1259a9e644cdSLuigi Rizzo bool notify; 1260a9e644cdSLuigi Rizzo 1261b6e66be2SVincenzo Maffione notify = netmap_pt_guest_txsync(pq->atok, pq->ktoa, kring, flags); 1262a9e644cdSLuigi Rizzo if (notify) { 1263a9e644cdSLuigi Rizzo ptnet_kick(pq); 1264a9e644cdSLuigi Rizzo } 1265a9e644cdSLuigi Rizzo 1266a9e644cdSLuigi Rizzo return 0; 1267a9e644cdSLuigi Rizzo } 1268a9e644cdSLuigi Rizzo 1269a9e644cdSLuigi Rizzo static int 1270a9e644cdSLuigi Rizzo ptnet_nm_rxsync(struct netmap_kring *kring, int flags) 1271a9e644cdSLuigi Rizzo { 1272a9e644cdSLuigi Rizzo struct ptnet_softc *sc = if_getsoftc(kring->na->ifp); 1273a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->rxqueues + kring->ring_id; 1274a9e644cdSLuigi Rizzo bool notify; 1275a9e644cdSLuigi Rizzo 1276b6e66be2SVincenzo Maffione notify = netmap_pt_guest_rxsync(pq->atok, pq->ktoa, kring, flags); 1277a9e644cdSLuigi Rizzo if (notify) { 1278a9e644cdSLuigi Rizzo ptnet_kick(pq); 1279a9e644cdSLuigi Rizzo } 1280a9e644cdSLuigi Rizzo 1281a9e644cdSLuigi Rizzo return 0; 1282a9e644cdSLuigi Rizzo } 1283a9e644cdSLuigi Rizzo 1284a9e644cdSLuigi Rizzo static void 12854f80b14cSVincenzo Maffione ptnet_nm_intr(struct netmap_adapter *na, int onoff) 12864f80b14cSVincenzo Maffione { 12874f80b14cSVincenzo Maffione struct ptnet_softc *sc = if_getsoftc(na->ifp); 12884f80b14cSVincenzo Maffione int i; 12894f80b14cSVincenzo Maffione 12904f80b14cSVincenzo Maffione for (i = 0; i < sc->num_rings; i++) { 12914f80b14cSVincenzo Maffione struct ptnet_queue *pq = sc->queues + i; 1292b6e66be2SVincenzo Maffione pq->atok->appl_need_kick = onoff; 12934f80b14cSVincenzo Maffione } 12944f80b14cSVincenzo Maffione } 12954f80b14cSVincenzo Maffione 12964f80b14cSVincenzo Maffione static void 1297a9e644cdSLuigi Rizzo ptnet_tx_intr(void *opaque) 1298a9e644cdSLuigi Rizzo { 1299a9e644cdSLuigi Rizzo struct ptnet_queue *pq = opaque; 1300a9e644cdSLuigi Rizzo struct ptnet_softc *sc = pq->sc; 1301a9e644cdSLuigi Rizzo 1302a9e644cdSLuigi Rizzo DBG(device_printf(sc->dev, "Tx interrupt #%d\n", pq->kring_id)); 1303a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS 1304a9e644cdSLuigi Rizzo pq->stats.intrs ++; 1305a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */ 1306a9e644cdSLuigi Rizzo 1307a9e644cdSLuigi Rizzo if (netmap_tx_irq(sc->ifp, pq->kring_id) != NM_IRQ_PASS) { 1308a9e644cdSLuigi Rizzo return; 1309a9e644cdSLuigi Rizzo } 1310a9e644cdSLuigi Rizzo 1311a9e644cdSLuigi Rizzo /* Schedule the tasqueue to flush process transmissions requests. 1312a9e644cdSLuigi Rizzo * However, vtnet, if_em and if_igb just call ptnet_transmit() here, 1313a9e644cdSLuigi Rizzo * at least when using MSI-X interrupts. The if_em driver, instead 1314a9e644cdSLuigi Rizzo * schedule taskqueue when using legacy interrupts. */ 1315a9e644cdSLuigi Rizzo taskqueue_enqueue(pq->taskq, &pq->task); 1316a9e644cdSLuigi Rizzo } 1317a9e644cdSLuigi Rizzo 1318a9e644cdSLuigi Rizzo static void 1319a9e644cdSLuigi Rizzo ptnet_rx_intr(void *opaque) 1320a9e644cdSLuigi Rizzo { 1321a9e644cdSLuigi Rizzo struct ptnet_queue *pq = opaque; 1322a9e644cdSLuigi Rizzo struct ptnet_softc *sc = pq->sc; 1323a9e644cdSLuigi Rizzo unsigned int unused; 1324a9e644cdSLuigi Rizzo 1325a9e644cdSLuigi Rizzo DBG(device_printf(sc->dev, "Rx interrupt #%d\n", pq->kring_id)); 1326a9e644cdSLuigi Rizzo #ifdef PTNETMAP_STATS 1327a9e644cdSLuigi Rizzo pq->stats.intrs ++; 1328a9e644cdSLuigi Rizzo #endif /* PTNETMAP_STATS */ 1329a9e644cdSLuigi Rizzo 1330a9e644cdSLuigi Rizzo if (netmap_rx_irq(sc->ifp, pq->kring_id, &unused) != NM_IRQ_PASS) { 1331a9e644cdSLuigi Rizzo return; 1332a9e644cdSLuigi Rizzo } 1333a9e644cdSLuigi Rizzo 1334a9e644cdSLuigi Rizzo /* Like vtnet, if_igb and if_em drivers when using MSI-X interrupts, 1335a9e644cdSLuigi Rizzo * receive-side processing is executed directly in the interrupt 1336a9e644cdSLuigi Rizzo * service routine. Alternatively, we may schedule the taskqueue. */ 1337a9e644cdSLuigi Rizzo ptnet_rx_eof(pq, PTNET_RX_BUDGET, true); 1338a9e644cdSLuigi Rizzo } 1339a9e644cdSLuigi Rizzo 1340a9e644cdSLuigi Rizzo static void 1341a9e644cdSLuigi Rizzo ptnet_vlan_tag_remove(struct mbuf *m) 1342a9e644cdSLuigi Rizzo { 1343a9e644cdSLuigi Rizzo struct ether_vlan_header *evh; 1344a9e644cdSLuigi Rizzo 1345a9e644cdSLuigi Rizzo evh = mtod(m, struct ether_vlan_header *); 1346a9e644cdSLuigi Rizzo m->m_pkthdr.ether_vtag = ntohs(evh->evl_tag); 1347a9e644cdSLuigi Rizzo m->m_flags |= M_VLANTAG; 1348a9e644cdSLuigi Rizzo 1349a9e644cdSLuigi Rizzo /* Strip the 802.1Q header. */ 1350a9e644cdSLuigi Rizzo bcopy((char *) evh, (char *) evh + ETHER_VLAN_ENCAP_LEN, 1351a9e644cdSLuigi Rizzo ETHER_HDR_LEN - ETHER_TYPE_LEN); 1352a9e644cdSLuigi Rizzo m_adj(m, ETHER_VLAN_ENCAP_LEN); 1353a9e644cdSLuigi Rizzo } 1354a9e644cdSLuigi Rizzo 1355a9e644cdSLuigi Rizzo static void 1356a9e644cdSLuigi Rizzo ptnet_ring_update(struct ptnet_queue *pq, struct netmap_kring *kring, 1357a9e644cdSLuigi Rizzo unsigned int head, unsigned int sync_flags) 1358a9e644cdSLuigi Rizzo { 1359a9e644cdSLuigi Rizzo struct netmap_ring *ring = kring->ring; 1360b6e66be2SVincenzo Maffione struct nm_csb_atok *atok = pq->atok; 1361b6e66be2SVincenzo Maffione struct nm_csb_ktoa *ktoa = pq->ktoa; 1362a9e644cdSLuigi Rizzo 1363a9e644cdSLuigi Rizzo /* Some packets have been pushed to the netmap ring. We have 1364a9e644cdSLuigi Rizzo * to tell the host to process the new packets, updating cur 1365a9e644cdSLuigi Rizzo * and head in the CSB. */ 1366a9e644cdSLuigi Rizzo ring->head = ring->cur = head; 1367a9e644cdSLuigi Rizzo 1368a9e644cdSLuigi Rizzo /* Mimic nm_txsync_prologue/nm_rxsync_prologue. */ 1369a9e644cdSLuigi Rizzo kring->rcur = kring->rhead = head; 1370a9e644cdSLuigi Rizzo 1371f79ba6d7SVincenzo Maffione nm_sync_kloop_appl_write(atok, kring->rcur, kring->rhead); 1372a9e644cdSLuigi Rizzo 1373a9e644cdSLuigi Rizzo /* Kick the host if needed. */ 1374b6e66be2SVincenzo Maffione if (NM_ACCESS_ONCE(ktoa->kern_need_kick)) { 1375b6e66be2SVincenzo Maffione atok->sync_flags = sync_flags; 1376a9e644cdSLuigi Rizzo ptnet_kick(pq); 1377a9e644cdSLuigi Rizzo } 1378a9e644cdSLuigi Rizzo } 1379a9e644cdSLuigi Rizzo 1380a9e644cdSLuigi Rizzo #define PTNET_TX_NOSPACE(_h, _k, _min) \ 1381a9e644cdSLuigi Rizzo ((((_h) < (_k)->rtail) ? 0 : (_k)->nkr_num_slots) + \ 1382a9e644cdSLuigi Rizzo (_k)->rtail - (_h)) < (_min) 1383a9e644cdSLuigi Rizzo 1384a9e644cdSLuigi Rizzo /* This function may be called by the network stack, or by 1385a9e644cdSLuigi Rizzo * by the taskqueue thread. */ 1386a9e644cdSLuigi Rizzo static int 1387a9e644cdSLuigi Rizzo ptnet_drain_transmit_queue(struct ptnet_queue *pq, unsigned int budget, 1388a9e644cdSLuigi Rizzo bool may_resched) 1389a9e644cdSLuigi Rizzo { 1390a9e644cdSLuigi Rizzo struct ptnet_softc *sc = pq->sc; 1391a9e644cdSLuigi Rizzo bool have_vnet_hdr = sc->vnet_hdr_len; 1392a9e644cdSLuigi Rizzo struct netmap_adapter *na = &sc->ptna->dr.up; 1393a9e644cdSLuigi Rizzo if_t ifp = sc->ifp; 1394a9e644cdSLuigi Rizzo unsigned int batch_count = 0; 1395b6e66be2SVincenzo Maffione struct nm_csb_atok *atok; 1396b6e66be2SVincenzo Maffione struct nm_csb_ktoa *ktoa; 1397a9e644cdSLuigi Rizzo struct netmap_kring *kring; 1398a9e644cdSLuigi Rizzo struct netmap_ring *ring; 1399a9e644cdSLuigi Rizzo struct netmap_slot *slot; 1400a9e644cdSLuigi Rizzo unsigned int count = 0; 1401a9e644cdSLuigi Rizzo unsigned int minspace; 1402a9e644cdSLuigi Rizzo unsigned int head; 1403a9e644cdSLuigi Rizzo unsigned int lim; 1404a9e644cdSLuigi Rizzo struct mbuf *mhead; 1405a9e644cdSLuigi Rizzo struct mbuf *mf; 1406a9e644cdSLuigi Rizzo int nmbuf_bytes; 1407a9e644cdSLuigi Rizzo uint8_t *nmbuf; 1408a9e644cdSLuigi Rizzo 1409a9e644cdSLuigi Rizzo if (!PTNET_Q_TRYLOCK(pq)) { 1410a9e644cdSLuigi Rizzo /* We failed to acquire the lock, schedule the taskqueue. */ 141175f4f3edSVincenzo Maffione nm_prlim(1, "Deferring TX work"); 1412a9e644cdSLuigi Rizzo if (may_resched) { 1413a9e644cdSLuigi Rizzo taskqueue_enqueue(pq->taskq, &pq->task); 1414a9e644cdSLuigi Rizzo } 1415a9e644cdSLuigi Rizzo 1416a9e644cdSLuigi Rizzo return 0; 1417a9e644cdSLuigi Rizzo } 1418a9e644cdSLuigi Rizzo 1419a9e644cdSLuigi Rizzo if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) { 1420a9e644cdSLuigi Rizzo PTNET_Q_UNLOCK(pq); 142175f4f3edSVincenzo Maffione nm_prlim(1, "Interface is down"); 1422a9e644cdSLuigi Rizzo return ENETDOWN; 1423a9e644cdSLuigi Rizzo } 1424a9e644cdSLuigi Rizzo 1425b6e66be2SVincenzo Maffione atok = pq->atok; 1426b6e66be2SVincenzo Maffione ktoa = pq->ktoa; 14272ff91c17SVincenzo Maffione kring = na->tx_rings[pq->kring_id]; 1428a9e644cdSLuigi Rizzo ring = kring->ring; 1429a9e644cdSLuigi Rizzo lim = kring->nkr_num_slots - 1; 1430a9e644cdSLuigi Rizzo head = ring->head; 1431a9e644cdSLuigi Rizzo minspace = sc->min_tx_space; 1432a9e644cdSLuigi Rizzo 1433a9e644cdSLuigi Rizzo while (count < budget) { 1434a9e644cdSLuigi Rizzo if (PTNET_TX_NOSPACE(head, kring, minspace)) { 1435a9e644cdSLuigi Rizzo /* We ran out of slot, let's see if the host has 1436a9e644cdSLuigi Rizzo * freed up some, by reading hwcur and hwtail from 1437a9e644cdSLuigi Rizzo * the CSB. */ 1438b6e66be2SVincenzo Maffione ptnet_sync_tail(ktoa, kring); 1439a9e644cdSLuigi Rizzo 1440a9e644cdSLuigi Rizzo if (PTNET_TX_NOSPACE(head, kring, minspace)) { 1441a9e644cdSLuigi Rizzo /* Still no slots available. Reactivate the 1442a9e644cdSLuigi Rizzo * interrupts so that we can be notified 1443a9e644cdSLuigi Rizzo * when some free slots are made available by 1444a9e644cdSLuigi Rizzo * the host. */ 1445b6e66be2SVincenzo Maffione atok->appl_need_kick = 1; 1446a9e644cdSLuigi Rizzo 1447f79ba6d7SVincenzo Maffione /* Double check. We need a full barrier to 1448f79ba6d7SVincenzo Maffione * prevent the store to atok->appl_need_kick 1449f79ba6d7SVincenzo Maffione * to be reordered with the load from 1450f79ba6d7SVincenzo Maffione * ktoa->hwcur and ktoa->hwtail (store-load 1451f79ba6d7SVincenzo Maffione * barrier). */ 1452f79ba6d7SVincenzo Maffione nm_stld_barrier(); 1453b6e66be2SVincenzo Maffione ptnet_sync_tail(ktoa, kring); 1454a9e644cdSLuigi Rizzo if (likely(PTNET_TX_NOSPACE(head, kring, 1455a9e644cdSLuigi Rizzo minspace))) { 1456a9e644cdSLuigi Rizzo break; 1457a9e644cdSLuigi Rizzo } 1458a9e644cdSLuigi Rizzo 145975f4f3edSVincenzo Maffione nm_prlim(1, "Found more slots by doublecheck"); 1460a9e644cdSLuigi Rizzo /* More slots were freed before reactivating 1461a9e644cdSLuigi Rizzo * the interrupts. */ 1462b6e66be2SVincenzo Maffione atok->appl_need_kick = 0; 1463a9e644cdSLuigi Rizzo } 1464a9e644cdSLuigi Rizzo } 1465a9e644cdSLuigi Rizzo 1466a9e644cdSLuigi Rizzo mhead = drbr_peek(ifp, pq->bufring); 1467a9e644cdSLuigi Rizzo if (!mhead) { 1468a9e644cdSLuigi Rizzo break; 1469a9e644cdSLuigi Rizzo } 1470a9e644cdSLuigi Rizzo 1471a9e644cdSLuigi Rizzo /* Initialize transmission state variables. */ 1472a9e644cdSLuigi Rizzo slot = ring->slot + head; 1473a9e644cdSLuigi Rizzo nmbuf = NMB(na, slot); 1474a9e644cdSLuigi Rizzo nmbuf_bytes = 0; 1475a9e644cdSLuigi Rizzo 1476a9e644cdSLuigi Rizzo /* If needed, prepare the virtio-net header at the beginning 1477a9e644cdSLuigi Rizzo * of the first slot. */ 1478a9e644cdSLuigi Rizzo if (have_vnet_hdr) { 1479a9e644cdSLuigi Rizzo struct virtio_net_hdr *vh = 1480a9e644cdSLuigi Rizzo (struct virtio_net_hdr *)nmbuf; 1481a9e644cdSLuigi Rizzo 1482a9e644cdSLuigi Rizzo /* For performance, we could replace this memset() with 1483a9e644cdSLuigi Rizzo * two 8-bytes-wide writes. */ 1484a9e644cdSLuigi Rizzo memset(nmbuf, 0, PTNET_HDR_SIZE); 1485a9e644cdSLuigi Rizzo if (mhead->m_pkthdr.csum_flags & PTNET_ALL_OFFLOAD) { 1486f8bc74e2SVincenzo Maffione mhead = virtio_net_tx_offload(ifp, mhead, false, 1487a9e644cdSLuigi Rizzo vh); 1488a9e644cdSLuigi Rizzo if (unlikely(!mhead)) { 1489a9e644cdSLuigi Rizzo /* Packet dropped because errors 1490a9e644cdSLuigi Rizzo * occurred while preparing the vnet 1491a9e644cdSLuigi Rizzo * header. Let's go ahead with the next 1492a9e644cdSLuigi Rizzo * packet. */ 1493a9e644cdSLuigi Rizzo pq->stats.errors ++; 1494a9e644cdSLuigi Rizzo drbr_advance(ifp, pq->bufring); 1495a9e644cdSLuigi Rizzo continue; 1496a9e644cdSLuigi Rizzo } 1497a9e644cdSLuigi Rizzo } 149875f4f3edSVincenzo Maffione nm_prdis(1, "%s: [csum_flags %lX] vnet hdr: flags %x " 1499a9e644cdSLuigi Rizzo "csum_start %u csum_ofs %u hdr_len = %u " 1500a9e644cdSLuigi Rizzo "gso_size %u gso_type %x", __func__, 1501a9e644cdSLuigi Rizzo mhead->m_pkthdr.csum_flags, vh->flags, 1502a9e644cdSLuigi Rizzo vh->csum_start, vh->csum_offset, vh->hdr_len, 1503a9e644cdSLuigi Rizzo vh->gso_size, vh->gso_type); 1504a9e644cdSLuigi Rizzo 1505a9e644cdSLuigi Rizzo nmbuf += PTNET_HDR_SIZE; 1506a9e644cdSLuigi Rizzo nmbuf_bytes += PTNET_HDR_SIZE; 1507a9e644cdSLuigi Rizzo } 1508a9e644cdSLuigi Rizzo 1509a9e644cdSLuigi Rizzo for (mf = mhead; mf; mf = mf->m_next) { 1510a9e644cdSLuigi Rizzo uint8_t *mdata = mf->m_data; 1511a9e644cdSLuigi Rizzo int mlen = mf->m_len; 1512a9e644cdSLuigi Rizzo 1513a9e644cdSLuigi Rizzo for (;;) { 1514a9e644cdSLuigi Rizzo int copy = NETMAP_BUF_SIZE(na) - nmbuf_bytes; 1515a9e644cdSLuigi Rizzo 1516a9e644cdSLuigi Rizzo if (mlen < copy) { 1517a9e644cdSLuigi Rizzo copy = mlen; 1518a9e644cdSLuigi Rizzo } 1519a9e644cdSLuigi Rizzo memcpy(nmbuf, mdata, copy); 1520a9e644cdSLuigi Rizzo 1521a9e644cdSLuigi Rizzo mdata += copy; 1522a9e644cdSLuigi Rizzo mlen -= copy; 1523a9e644cdSLuigi Rizzo nmbuf += copy; 1524a9e644cdSLuigi Rizzo nmbuf_bytes += copy; 1525a9e644cdSLuigi Rizzo 1526a9e644cdSLuigi Rizzo if (!mlen) { 1527a9e644cdSLuigi Rizzo break; 1528a9e644cdSLuigi Rizzo } 1529a9e644cdSLuigi Rizzo 1530a9e644cdSLuigi Rizzo slot->len = nmbuf_bytes; 1531a9e644cdSLuigi Rizzo slot->flags = NS_MOREFRAG; 1532a9e644cdSLuigi Rizzo 1533a9e644cdSLuigi Rizzo head = nm_next(head, lim); 1534a9e644cdSLuigi Rizzo KASSERT(head != ring->tail, 1535a9e644cdSLuigi Rizzo ("Unexpectedly run out of TX space")); 1536a9e644cdSLuigi Rizzo slot = ring->slot + head; 1537a9e644cdSLuigi Rizzo nmbuf = NMB(na, slot); 1538a9e644cdSLuigi Rizzo nmbuf_bytes = 0; 1539a9e644cdSLuigi Rizzo } 1540a9e644cdSLuigi Rizzo } 1541a9e644cdSLuigi Rizzo 1542a9e644cdSLuigi Rizzo /* Complete last slot and update head. */ 1543a9e644cdSLuigi Rizzo slot->len = nmbuf_bytes; 1544a9e644cdSLuigi Rizzo slot->flags = 0; 1545a9e644cdSLuigi Rizzo head = nm_next(head, lim); 1546a9e644cdSLuigi Rizzo 1547a9e644cdSLuigi Rizzo /* Consume the packet just processed. */ 1548a9e644cdSLuigi Rizzo drbr_advance(ifp, pq->bufring); 1549a9e644cdSLuigi Rizzo 1550a9e644cdSLuigi Rizzo /* Copy the packet to listeners. */ 1551a9e644cdSLuigi Rizzo ETHER_BPF_MTAP(ifp, mhead); 1552a9e644cdSLuigi Rizzo 1553a9e644cdSLuigi Rizzo pq->stats.packets ++; 1554a9e644cdSLuigi Rizzo pq->stats.bytes += mhead->m_pkthdr.len; 1555a9e644cdSLuigi Rizzo if (mhead->m_flags & M_MCAST) { 1556a9e644cdSLuigi Rizzo pq->stats.mcasts ++; 1557a9e644cdSLuigi Rizzo } 1558a9e644cdSLuigi Rizzo 1559a9e644cdSLuigi Rizzo m_freem(mhead); 1560a9e644cdSLuigi Rizzo 1561a9e644cdSLuigi Rizzo count ++; 1562a9e644cdSLuigi Rizzo if (++batch_count == PTNET_TX_BATCH) { 1563a9e644cdSLuigi Rizzo ptnet_ring_update(pq, kring, head, NAF_FORCE_RECLAIM); 1564a9e644cdSLuigi Rizzo batch_count = 0; 1565a9e644cdSLuigi Rizzo } 1566a9e644cdSLuigi Rizzo } 1567a9e644cdSLuigi Rizzo 1568a9e644cdSLuigi Rizzo if (batch_count) { 1569a9e644cdSLuigi Rizzo ptnet_ring_update(pq, kring, head, NAF_FORCE_RECLAIM); 1570a9e644cdSLuigi Rizzo } 1571a9e644cdSLuigi Rizzo 1572a9e644cdSLuigi Rizzo if (count >= budget && may_resched) { 157375f4f3edSVincenzo Maffione DBG(nm_prlim(1, "out of budget: resched, %d mbufs pending\n", 1574a9e644cdSLuigi Rizzo drbr_inuse(ifp, pq->bufring))); 1575a9e644cdSLuigi Rizzo taskqueue_enqueue(pq->taskq, &pq->task); 1576a9e644cdSLuigi Rizzo } 1577a9e644cdSLuigi Rizzo 1578a9e644cdSLuigi Rizzo PTNET_Q_UNLOCK(pq); 1579a9e644cdSLuigi Rizzo 1580a9e644cdSLuigi Rizzo return count; 1581a9e644cdSLuigi Rizzo } 1582a9e644cdSLuigi Rizzo 1583a9e644cdSLuigi Rizzo static int 1584a9e644cdSLuigi Rizzo ptnet_transmit(if_t ifp, struct mbuf *m) 1585a9e644cdSLuigi Rizzo { 1586a9e644cdSLuigi Rizzo struct ptnet_softc *sc = if_getsoftc(ifp); 1587a9e644cdSLuigi Rizzo struct ptnet_queue *pq; 1588a9e644cdSLuigi Rizzo unsigned int queue_idx; 1589a9e644cdSLuigi Rizzo int err; 1590a9e644cdSLuigi Rizzo 1591a9e644cdSLuigi Rizzo DBG(device_printf(sc->dev, "transmit %p\n", m)); 1592a9e644cdSLuigi Rizzo 1593a9e644cdSLuigi Rizzo /* Insert 802.1Q header if needed. */ 1594a9e644cdSLuigi Rizzo if (m->m_flags & M_VLANTAG) { 1595a9e644cdSLuigi Rizzo m = ether_vlanencap(m, m->m_pkthdr.ether_vtag); 1596a9e644cdSLuigi Rizzo if (m == NULL) { 1597a9e644cdSLuigi Rizzo return ENOBUFS; 1598a9e644cdSLuigi Rizzo } 1599a9e644cdSLuigi Rizzo m->m_flags &= ~M_VLANTAG; 1600a9e644cdSLuigi Rizzo } 1601a9e644cdSLuigi Rizzo 1602a9e644cdSLuigi Rizzo /* Get the flow-id if available. */ 1603a9e644cdSLuigi Rizzo queue_idx = (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) ? 1604a9e644cdSLuigi Rizzo m->m_pkthdr.flowid : curcpu; 1605a9e644cdSLuigi Rizzo 1606a9e644cdSLuigi Rizzo if (unlikely(queue_idx >= sc->num_tx_rings)) { 1607a9e644cdSLuigi Rizzo queue_idx %= sc->num_tx_rings; 1608a9e644cdSLuigi Rizzo } 1609a9e644cdSLuigi Rizzo 1610a9e644cdSLuigi Rizzo pq = sc->queues + queue_idx; 1611a9e644cdSLuigi Rizzo 1612a9e644cdSLuigi Rizzo err = drbr_enqueue(ifp, pq->bufring, m); 1613a9e644cdSLuigi Rizzo if (err) { 1614a9e644cdSLuigi Rizzo /* ENOBUFS when the bufring is full */ 161575f4f3edSVincenzo Maffione nm_prlim(1, "%s: drbr_enqueue() failed %d\n", 1616a9e644cdSLuigi Rizzo __func__, err); 1617a9e644cdSLuigi Rizzo pq->stats.errors ++; 1618a9e644cdSLuigi Rizzo return err; 1619a9e644cdSLuigi Rizzo } 1620a9e644cdSLuigi Rizzo 1621a9e644cdSLuigi Rizzo if (ifp->if_capenable & IFCAP_POLLING) { 1622a9e644cdSLuigi Rizzo /* If polling is on, the transmit queues will be 1623a9e644cdSLuigi Rizzo * drained by the poller. */ 1624a9e644cdSLuigi Rizzo return 0; 1625a9e644cdSLuigi Rizzo } 1626a9e644cdSLuigi Rizzo 1627a9e644cdSLuigi Rizzo err = ptnet_drain_transmit_queue(pq, PTNET_TX_BUDGET, true); 1628a9e644cdSLuigi Rizzo 1629a9e644cdSLuigi Rizzo return (err < 0) ? err : 0; 1630a9e644cdSLuigi Rizzo } 1631a9e644cdSLuigi Rizzo 1632a9e644cdSLuigi Rizzo static unsigned int 1633a9e644cdSLuigi Rizzo ptnet_rx_discard(struct netmap_kring *kring, unsigned int head) 1634a9e644cdSLuigi Rizzo { 1635a9e644cdSLuigi Rizzo struct netmap_ring *ring = kring->ring; 1636a9e644cdSLuigi Rizzo struct netmap_slot *slot = ring->slot + head; 1637a9e644cdSLuigi Rizzo 1638a9e644cdSLuigi Rizzo for (;;) { 1639a9e644cdSLuigi Rizzo head = nm_next(head, kring->nkr_num_slots - 1); 1640a9e644cdSLuigi Rizzo if (!(slot->flags & NS_MOREFRAG) || head == ring->tail) { 1641a9e644cdSLuigi Rizzo break; 1642a9e644cdSLuigi Rizzo } 1643a9e644cdSLuigi Rizzo slot = ring->slot + head; 1644a9e644cdSLuigi Rizzo } 1645a9e644cdSLuigi Rizzo 1646a9e644cdSLuigi Rizzo return head; 1647a9e644cdSLuigi Rizzo } 1648a9e644cdSLuigi Rizzo 1649a9e644cdSLuigi Rizzo static inline struct mbuf * 1650a9e644cdSLuigi Rizzo ptnet_rx_slot(struct mbuf *mtail, uint8_t *nmbuf, unsigned int nmbuf_len) 1651a9e644cdSLuigi Rizzo { 1652a9e644cdSLuigi Rizzo uint8_t *mdata = mtod(mtail, uint8_t *) + mtail->m_len; 1653a9e644cdSLuigi Rizzo 1654a9e644cdSLuigi Rizzo do { 1655a9e644cdSLuigi Rizzo unsigned int copy; 1656a9e644cdSLuigi Rizzo 1657a9e644cdSLuigi Rizzo if (mtail->m_len == MCLBYTES) { 1658a9e644cdSLuigi Rizzo struct mbuf *mf; 1659a9e644cdSLuigi Rizzo 1660a9e644cdSLuigi Rizzo mf = m_getcl(M_NOWAIT, MT_DATA, 0); 1661a9e644cdSLuigi Rizzo if (unlikely(!mf)) { 1662a9e644cdSLuigi Rizzo return NULL; 1663a9e644cdSLuigi Rizzo } 1664a9e644cdSLuigi Rizzo 1665a9e644cdSLuigi Rizzo mtail->m_next = mf; 1666a9e644cdSLuigi Rizzo mtail = mf; 1667a9e644cdSLuigi Rizzo mdata = mtod(mtail, uint8_t *); 1668a9e644cdSLuigi Rizzo mtail->m_len = 0; 1669a9e644cdSLuigi Rizzo } 1670a9e644cdSLuigi Rizzo 1671a9e644cdSLuigi Rizzo copy = MCLBYTES - mtail->m_len; 1672a9e644cdSLuigi Rizzo if (nmbuf_len < copy) { 1673a9e644cdSLuigi Rizzo copy = nmbuf_len; 1674a9e644cdSLuigi Rizzo } 1675a9e644cdSLuigi Rizzo 1676a9e644cdSLuigi Rizzo memcpy(mdata, nmbuf, copy); 1677a9e644cdSLuigi Rizzo 1678a9e644cdSLuigi Rizzo nmbuf += copy; 1679a9e644cdSLuigi Rizzo nmbuf_len -= copy; 1680a9e644cdSLuigi Rizzo mdata += copy; 1681a9e644cdSLuigi Rizzo mtail->m_len += copy; 1682a9e644cdSLuigi Rizzo } while (nmbuf_len); 1683a9e644cdSLuigi Rizzo 1684a9e644cdSLuigi Rizzo return mtail; 1685a9e644cdSLuigi Rizzo } 1686a9e644cdSLuigi Rizzo 1687a9e644cdSLuigi Rizzo static int 1688a9e644cdSLuigi Rizzo ptnet_rx_eof(struct ptnet_queue *pq, unsigned int budget, bool may_resched) 1689a9e644cdSLuigi Rizzo { 1690a9e644cdSLuigi Rizzo struct ptnet_softc *sc = pq->sc; 1691a9e644cdSLuigi Rizzo bool have_vnet_hdr = sc->vnet_hdr_len; 1692b6e66be2SVincenzo Maffione struct nm_csb_atok *atok = pq->atok; 1693b6e66be2SVincenzo Maffione struct nm_csb_ktoa *ktoa = pq->ktoa; 1694a9e644cdSLuigi Rizzo struct netmap_adapter *na = &sc->ptna->dr.up; 16952ff91c17SVincenzo Maffione struct netmap_kring *kring = na->rx_rings[pq->kring_id]; 1696a9e644cdSLuigi Rizzo struct netmap_ring *ring = kring->ring; 1697a9e644cdSLuigi Rizzo unsigned int const lim = kring->nkr_num_slots - 1; 1698a9e644cdSLuigi Rizzo unsigned int batch_count = 0; 1699a9e644cdSLuigi Rizzo if_t ifp = sc->ifp; 1700a9e644cdSLuigi Rizzo unsigned int count = 0; 170146023447SVincenzo Maffione uint32_t head; 1702a9e644cdSLuigi Rizzo 1703a9e644cdSLuigi Rizzo PTNET_Q_LOCK(pq); 1704a9e644cdSLuigi Rizzo 1705a9e644cdSLuigi Rizzo if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) { 1706a9e644cdSLuigi Rizzo goto unlock; 1707a9e644cdSLuigi Rizzo } 1708a9e644cdSLuigi Rizzo 1709a9e644cdSLuigi Rizzo kring->nr_kflags &= ~NKR_PENDINTR; 1710a9e644cdSLuigi Rizzo 171146023447SVincenzo Maffione head = ring->head; 1712a9e644cdSLuigi Rizzo while (count < budget) { 171346023447SVincenzo Maffione uint32_t prev_head = head; 1714a9e644cdSLuigi Rizzo struct mbuf *mhead, *mtail; 1715a9e644cdSLuigi Rizzo struct virtio_net_hdr *vh; 1716a9e644cdSLuigi Rizzo struct netmap_slot *slot; 1717a9e644cdSLuigi Rizzo unsigned int nmbuf_len; 1718a9e644cdSLuigi Rizzo uint8_t *nmbuf; 171946023447SVincenzo Maffione int deliver = 1; /* the mbuf to the network stack. */ 1720a9e644cdSLuigi Rizzo host_sync: 1721a9e644cdSLuigi Rizzo if (head == ring->tail) { 1722a9e644cdSLuigi Rizzo /* We ran out of slot, let's see if the host has 1723a9e644cdSLuigi Rizzo * added some, by reading hwcur and hwtail from 1724a9e644cdSLuigi Rizzo * the CSB. */ 1725b6e66be2SVincenzo Maffione ptnet_sync_tail(ktoa, kring); 1726a9e644cdSLuigi Rizzo 1727a9e644cdSLuigi Rizzo if (head == ring->tail) { 1728a9e644cdSLuigi Rizzo /* Still no slots available. Reactivate 1729a9e644cdSLuigi Rizzo * interrupts as they were disabled by the 1730a9e644cdSLuigi Rizzo * host thread right before issuing the 1731a9e644cdSLuigi Rizzo * last interrupt. */ 1732b6e66be2SVincenzo Maffione atok->appl_need_kick = 1; 1733a9e644cdSLuigi Rizzo 1734f79ba6d7SVincenzo Maffione /* Double check for more completed RX slots. 1735f79ba6d7SVincenzo Maffione * We need a full barrier to prevent the store 1736f79ba6d7SVincenzo Maffione * to atok->appl_need_kick to be reordered with 1737f79ba6d7SVincenzo Maffione * the load from ktoa->hwcur and ktoa->hwtail 1738f79ba6d7SVincenzo Maffione * (store-load barrier). */ 1739f79ba6d7SVincenzo Maffione nm_stld_barrier(); 1740b6e66be2SVincenzo Maffione ptnet_sync_tail(ktoa, kring); 1741a9e644cdSLuigi Rizzo if (likely(head == ring->tail)) { 1742a9e644cdSLuigi Rizzo break; 1743a9e644cdSLuigi Rizzo } 1744b6e66be2SVincenzo Maffione atok->appl_need_kick = 0; 1745a9e644cdSLuigi Rizzo } 1746a9e644cdSLuigi Rizzo } 1747a9e644cdSLuigi Rizzo 1748a9e644cdSLuigi Rizzo /* Initialize ring state variables, possibly grabbing the 1749a9e644cdSLuigi Rizzo * virtio-net header. */ 1750a9e644cdSLuigi Rizzo slot = ring->slot + head; 1751a9e644cdSLuigi Rizzo nmbuf = NMB(na, slot); 1752a9e644cdSLuigi Rizzo nmbuf_len = slot->len; 1753a9e644cdSLuigi Rizzo 1754a9e644cdSLuigi Rizzo vh = (struct virtio_net_hdr *)nmbuf; 1755a9e644cdSLuigi Rizzo if (have_vnet_hdr) { 1756a9e644cdSLuigi Rizzo if (unlikely(nmbuf_len < PTNET_HDR_SIZE)) { 1757a9e644cdSLuigi Rizzo /* There is no good reason why host should 1758a9e644cdSLuigi Rizzo * put the header in multiple netmap slots. 1759a9e644cdSLuigi Rizzo * If this is the case, discard. */ 176075f4f3edSVincenzo Maffione nm_prlim(1, "Fragmented vnet-hdr: dropping"); 1761a9e644cdSLuigi Rizzo head = ptnet_rx_discard(kring, head); 1762a9e644cdSLuigi Rizzo pq->stats.iqdrops ++; 176346023447SVincenzo Maffione deliver = 0; 1764a9e644cdSLuigi Rizzo goto skip; 1765a9e644cdSLuigi Rizzo } 176675f4f3edSVincenzo Maffione nm_prdis(1, "%s: vnet hdr: flags %x csum_start %u " 1767a9e644cdSLuigi Rizzo "csum_ofs %u hdr_len = %u gso_size %u " 1768a9e644cdSLuigi Rizzo "gso_type %x", __func__, vh->flags, 1769a9e644cdSLuigi Rizzo vh->csum_start, vh->csum_offset, vh->hdr_len, 1770a9e644cdSLuigi Rizzo vh->gso_size, vh->gso_type); 1771a9e644cdSLuigi Rizzo nmbuf += PTNET_HDR_SIZE; 1772a9e644cdSLuigi Rizzo nmbuf_len -= PTNET_HDR_SIZE; 1773a9e644cdSLuigi Rizzo } 1774a9e644cdSLuigi Rizzo 1775a9e644cdSLuigi Rizzo /* Allocate the head of a new mbuf chain. 1776a9e644cdSLuigi Rizzo * We use m_getcl() to allocate an mbuf with standard cluster 1777a9e644cdSLuigi Rizzo * size (MCLBYTES). In the future we could use m_getjcl() 1778a9e644cdSLuigi Rizzo * to choose different sizes. */ 1779a9e644cdSLuigi Rizzo mhead = mtail = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 1780a9e644cdSLuigi Rizzo if (unlikely(mhead == NULL)) { 1781a9e644cdSLuigi Rizzo device_printf(sc->dev, "%s: failed to allocate mbuf " 1782a9e644cdSLuigi Rizzo "head\n", __func__); 1783a9e644cdSLuigi Rizzo pq->stats.errors ++; 1784a9e644cdSLuigi Rizzo break; 1785a9e644cdSLuigi Rizzo } 1786a9e644cdSLuigi Rizzo 1787a9e644cdSLuigi Rizzo /* Initialize the mbuf state variables. */ 1788a9e644cdSLuigi Rizzo mhead->m_pkthdr.len = nmbuf_len; 1789a9e644cdSLuigi Rizzo mtail->m_len = 0; 1790a9e644cdSLuigi Rizzo 1791a9e644cdSLuigi Rizzo /* Scan all the netmap slots containing the current packet. */ 1792a9e644cdSLuigi Rizzo for (;;) { 1793a9e644cdSLuigi Rizzo DBG(device_printf(sc->dev, "%s: h %u t %u rcv frag " 1794a9e644cdSLuigi Rizzo "len %u, flags %u\n", __func__, 1795a9e644cdSLuigi Rizzo head, ring->tail, slot->len, 1796a9e644cdSLuigi Rizzo slot->flags)); 1797a9e644cdSLuigi Rizzo 1798a9e644cdSLuigi Rizzo mtail = ptnet_rx_slot(mtail, nmbuf, nmbuf_len); 1799a9e644cdSLuigi Rizzo if (unlikely(!mtail)) { 1800a9e644cdSLuigi Rizzo /* Ouch. We ran out of memory while processing 1801a9e644cdSLuigi Rizzo * a packet. We have to restore the previous 1802a9e644cdSLuigi Rizzo * head position, free the mbuf chain, and 1803a9e644cdSLuigi Rizzo * schedule the taskqueue to give the packet 1804a9e644cdSLuigi Rizzo * another chance. */ 1805a9e644cdSLuigi Rizzo device_printf(sc->dev, "%s: failed to allocate" 1806a9e644cdSLuigi Rizzo " mbuf frag, reset head %u --> %u\n", 1807a9e644cdSLuigi Rizzo __func__, head, prev_head); 1808a9e644cdSLuigi Rizzo head = prev_head; 1809a9e644cdSLuigi Rizzo m_freem(mhead); 1810a9e644cdSLuigi Rizzo pq->stats.errors ++; 1811a9e644cdSLuigi Rizzo if (may_resched) { 1812a9e644cdSLuigi Rizzo taskqueue_enqueue(pq->taskq, 1813a9e644cdSLuigi Rizzo &pq->task); 1814a9e644cdSLuigi Rizzo } 1815a9e644cdSLuigi Rizzo goto escape; 1816a9e644cdSLuigi Rizzo } 1817a9e644cdSLuigi Rizzo 1818a9e644cdSLuigi Rizzo /* We have to increment head irrespective of the 1819a9e644cdSLuigi Rizzo * NS_MOREFRAG being set or not. */ 1820a9e644cdSLuigi Rizzo head = nm_next(head, lim); 1821a9e644cdSLuigi Rizzo 1822a9e644cdSLuigi Rizzo if (!(slot->flags & NS_MOREFRAG)) { 1823a9e644cdSLuigi Rizzo break; 1824a9e644cdSLuigi Rizzo } 1825a9e644cdSLuigi Rizzo 1826a9e644cdSLuigi Rizzo if (unlikely(head == ring->tail)) { 1827a9e644cdSLuigi Rizzo /* The very last slot prepared by the host has 1828a9e644cdSLuigi Rizzo * the NS_MOREFRAG set. Drop it and continue 1829a9e644cdSLuigi Rizzo * the outer cycle (to do the double-check). */ 183075f4f3edSVincenzo Maffione nm_prlim(1, "Incomplete packet: dropping"); 1831a9e644cdSLuigi Rizzo m_freem(mhead); 1832a9e644cdSLuigi Rizzo pq->stats.iqdrops ++; 1833a9e644cdSLuigi Rizzo goto host_sync; 1834a9e644cdSLuigi Rizzo } 1835a9e644cdSLuigi Rizzo 1836a9e644cdSLuigi Rizzo slot = ring->slot + head; 1837a9e644cdSLuigi Rizzo nmbuf = NMB(na, slot); 1838a9e644cdSLuigi Rizzo nmbuf_len = slot->len; 1839a9e644cdSLuigi Rizzo mhead->m_pkthdr.len += nmbuf_len; 1840a9e644cdSLuigi Rizzo } 1841a9e644cdSLuigi Rizzo 1842a9e644cdSLuigi Rizzo mhead->m_pkthdr.rcvif = ifp; 1843a9e644cdSLuigi Rizzo mhead->m_pkthdr.csum_flags = 0; 1844a9e644cdSLuigi Rizzo 1845a9e644cdSLuigi Rizzo /* Store the queue idx in the packet header. */ 1846a9e644cdSLuigi Rizzo mhead->m_pkthdr.flowid = pq->kring_id; 1847a9e644cdSLuigi Rizzo M_HASHTYPE_SET(mhead, M_HASHTYPE_OPAQUE); 1848a9e644cdSLuigi Rizzo 1849a9e644cdSLuigi Rizzo if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { 1850a9e644cdSLuigi Rizzo struct ether_header *eh; 1851a9e644cdSLuigi Rizzo 1852a9e644cdSLuigi Rizzo eh = mtod(mhead, struct ether_header *); 1853a9e644cdSLuigi Rizzo if (eh->ether_type == htons(ETHERTYPE_VLAN)) { 1854a9e644cdSLuigi Rizzo ptnet_vlan_tag_remove(mhead); 1855a9e644cdSLuigi Rizzo /* 1856a9e644cdSLuigi Rizzo * With the 802.1Q header removed, update the 1857a9e644cdSLuigi Rizzo * checksum starting location accordingly. 1858a9e644cdSLuigi Rizzo */ 1859a9e644cdSLuigi Rizzo if (vh->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) 1860a9e644cdSLuigi Rizzo vh->csum_start -= ETHER_VLAN_ENCAP_LEN; 1861a9e644cdSLuigi Rizzo } 1862a9e644cdSLuigi Rizzo } 1863a9e644cdSLuigi Rizzo 1864f8bc74e2SVincenzo Maffione if (unlikely(have_vnet_hdr && virtio_net_rx_csum(mhead, vh))) { 1865a9e644cdSLuigi Rizzo m_freem(mhead); 186675f4f3edSVincenzo Maffione nm_prlim(1, "Csum offload error: dropping"); 1867a9e644cdSLuigi Rizzo pq->stats.iqdrops ++; 186846023447SVincenzo Maffione deliver = 0; 1869a9e644cdSLuigi Rizzo } 1870a9e644cdSLuigi Rizzo 187146023447SVincenzo Maffione skip: 187246023447SVincenzo Maffione count ++; 187346023447SVincenzo Maffione if (++batch_count >= PTNET_RX_BATCH) { 187446023447SVincenzo Maffione /* Some packets have been (or will be) pushed to the network 187546023447SVincenzo Maffione * stack. We need to update the CSB to tell the host about 187646023447SVincenzo Maffione * the new ring->cur and ring->head (RX buffer refill). */ 187746023447SVincenzo Maffione ptnet_ring_update(pq, kring, head, NAF_FORCE_READ); 187846023447SVincenzo Maffione batch_count = 0; 187946023447SVincenzo Maffione } 188046023447SVincenzo Maffione 188146023447SVincenzo Maffione if (likely(deliver)) { 1882a9e644cdSLuigi Rizzo pq->stats.packets ++; 1883a9e644cdSLuigi Rizzo pq->stats.bytes += mhead->m_pkthdr.len; 1884a9e644cdSLuigi Rizzo 1885a9e644cdSLuigi Rizzo PTNET_Q_UNLOCK(pq); 1886a9e644cdSLuigi Rizzo (*ifp->if_input)(ifp, mhead); 1887a9e644cdSLuigi Rizzo PTNET_Q_LOCK(pq); 188846023447SVincenzo Maffione /* The ring->head index (and related indices) are 188946023447SVincenzo Maffione * updated under pq lock by ptnet_ring_update(). 189046023447SVincenzo Maffione * Since we dropped the lock to call if_input(), we 189146023447SVincenzo Maffione * must reload ring->head and restart processing the 189246023447SVincenzo Maffione * ring from there. */ 189346023447SVincenzo Maffione head = ring->head; 1894a9e644cdSLuigi Rizzo 1895a9e644cdSLuigi Rizzo if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) { 1896a9e644cdSLuigi Rizzo /* The interface has gone down while we didn't 1897a9e644cdSLuigi Rizzo * have the lock. Stop any processing and exit. */ 1898a9e644cdSLuigi Rizzo goto unlock; 1899a9e644cdSLuigi Rizzo } 1900a9e644cdSLuigi Rizzo } 1901a9e644cdSLuigi Rizzo } 1902a9e644cdSLuigi Rizzo escape: 1903a9e644cdSLuigi Rizzo if (batch_count) { 1904a9e644cdSLuigi Rizzo ptnet_ring_update(pq, kring, head, NAF_FORCE_READ); 1905a9e644cdSLuigi Rizzo 1906a9e644cdSLuigi Rizzo } 1907a9e644cdSLuigi Rizzo 1908a9e644cdSLuigi Rizzo if (count >= budget && may_resched) { 1909a9e644cdSLuigi Rizzo /* If we ran out of budget or the double-check found new 1910a9e644cdSLuigi Rizzo * slots to process, schedule the taskqueue. */ 191175f4f3edSVincenzo Maffione DBG(nm_prlim(1, "out of budget: resched h %u t %u\n", 1912a9e644cdSLuigi Rizzo head, ring->tail)); 1913a9e644cdSLuigi Rizzo taskqueue_enqueue(pq->taskq, &pq->task); 1914a9e644cdSLuigi Rizzo } 1915a9e644cdSLuigi Rizzo unlock: 1916a9e644cdSLuigi Rizzo PTNET_Q_UNLOCK(pq); 1917a9e644cdSLuigi Rizzo 1918a9e644cdSLuigi Rizzo return count; 1919a9e644cdSLuigi Rizzo } 1920a9e644cdSLuigi Rizzo 1921a9e644cdSLuigi Rizzo static void 1922a9e644cdSLuigi Rizzo ptnet_rx_task(void *context, int pending) 1923a9e644cdSLuigi Rizzo { 1924a9e644cdSLuigi Rizzo struct ptnet_queue *pq = context; 1925a9e644cdSLuigi Rizzo 192675f4f3edSVincenzo Maffione DBG(nm_prlim(1, "%s: pq #%u\n", __func__, pq->kring_id)); 1927a9e644cdSLuigi Rizzo ptnet_rx_eof(pq, PTNET_RX_BUDGET, true); 1928a9e644cdSLuigi Rizzo } 1929a9e644cdSLuigi Rizzo 1930a9e644cdSLuigi Rizzo static void 1931a9e644cdSLuigi Rizzo ptnet_tx_task(void *context, int pending) 1932a9e644cdSLuigi Rizzo { 1933a9e644cdSLuigi Rizzo struct ptnet_queue *pq = context; 1934a9e644cdSLuigi Rizzo 193575f4f3edSVincenzo Maffione DBG(nm_prlim(1, "%s: pq #%u\n", __func__, pq->kring_id)); 1936a9e644cdSLuigi Rizzo ptnet_drain_transmit_queue(pq, PTNET_TX_BUDGET, true); 1937a9e644cdSLuigi Rizzo } 1938a9e644cdSLuigi Rizzo 1939a9e644cdSLuigi Rizzo #ifdef DEVICE_POLLING 1940a9e644cdSLuigi Rizzo /* We don't need to handle differently POLL_AND_CHECK_STATUS and 1941a9e644cdSLuigi Rizzo * POLL_ONLY, since we don't have an Interrupt Status Register. */ 1942a9e644cdSLuigi Rizzo static int 1943a9e644cdSLuigi Rizzo ptnet_poll(if_t ifp, enum poll_cmd cmd, int budget) 1944a9e644cdSLuigi Rizzo { 1945a9e644cdSLuigi Rizzo struct ptnet_softc *sc = if_getsoftc(ifp); 1946a9e644cdSLuigi Rizzo unsigned int queue_budget; 1947a9e644cdSLuigi Rizzo unsigned int count = 0; 1948a9e644cdSLuigi Rizzo bool borrow = false; 1949a9e644cdSLuigi Rizzo int i; 1950a9e644cdSLuigi Rizzo 1951a9e644cdSLuigi Rizzo KASSERT(sc->num_rings > 0, ("Found no queues in while polling ptnet")); 1952a9e644cdSLuigi Rizzo queue_budget = MAX(budget / sc->num_rings, 1); 195375f4f3edSVincenzo Maffione nm_prlim(1, "Per-queue budget is %d", queue_budget); 1954a9e644cdSLuigi Rizzo 1955a9e644cdSLuigi Rizzo while (budget) { 1956a9e644cdSLuigi Rizzo unsigned int rcnt = 0; 1957a9e644cdSLuigi Rizzo 1958a9e644cdSLuigi Rizzo for (i = 0; i < sc->num_rings; i++) { 1959a9e644cdSLuigi Rizzo struct ptnet_queue *pq = sc->queues + i; 1960a9e644cdSLuigi Rizzo 1961a9e644cdSLuigi Rizzo if (borrow) { 1962a9e644cdSLuigi Rizzo queue_budget = MIN(queue_budget, budget); 1963a9e644cdSLuigi Rizzo if (queue_budget == 0) { 1964a9e644cdSLuigi Rizzo break; 1965a9e644cdSLuigi Rizzo } 1966a9e644cdSLuigi Rizzo } 1967a9e644cdSLuigi Rizzo 1968a9e644cdSLuigi Rizzo if (i < sc->num_tx_rings) { 1969a9e644cdSLuigi Rizzo rcnt += ptnet_drain_transmit_queue(pq, 1970a9e644cdSLuigi Rizzo queue_budget, false); 1971a9e644cdSLuigi Rizzo } else { 1972a9e644cdSLuigi Rizzo rcnt += ptnet_rx_eof(pq, queue_budget, 1973a9e644cdSLuigi Rizzo false); 1974a9e644cdSLuigi Rizzo } 1975a9e644cdSLuigi Rizzo } 1976a9e644cdSLuigi Rizzo 1977a9e644cdSLuigi Rizzo if (!rcnt) { 1978a9e644cdSLuigi Rizzo /* A scan of the queues gave no result, we can 1979a9e644cdSLuigi Rizzo * stop here. */ 1980a9e644cdSLuigi Rizzo break; 1981a9e644cdSLuigi Rizzo } 1982a9e644cdSLuigi Rizzo 1983a9e644cdSLuigi Rizzo if (rcnt > budget) { 1984a9e644cdSLuigi Rizzo /* This may happen when initial budget < sc->num_rings, 1985a9e644cdSLuigi Rizzo * since one packet budget is given to each queue 1986a9e644cdSLuigi Rizzo * anyway. Just pretend we didn't eat "so much". */ 1987a9e644cdSLuigi Rizzo rcnt = budget; 1988a9e644cdSLuigi Rizzo } 1989a9e644cdSLuigi Rizzo count += rcnt; 1990a9e644cdSLuigi Rizzo budget -= rcnt; 1991a9e644cdSLuigi Rizzo borrow = true; 1992a9e644cdSLuigi Rizzo } 1993a9e644cdSLuigi Rizzo 1994a9e644cdSLuigi Rizzo 1995a9e644cdSLuigi Rizzo return count; 1996a9e644cdSLuigi Rizzo } 1997a9e644cdSLuigi Rizzo #endif /* DEVICE_POLLING */ 1998*2ec213abSVincenzo Maffione #endif /* WITH_PTNETMAP */ 1999