1 /* 2 * Copyright (C) 2014-2018 Vincenzo Maffione, Luigi Rizzo. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 /* 27 * $FreeBSD$ 28 */ 29 30 #include <net/netmap.h> 31 #include <sys/selinfo.h> 32 #include <vm/vm.h> 33 #include <vm/pmap.h> /* vtophys ? */ 34 #include <dev/netmap/netmap_kern.h> 35 36 /* Register and unregister. */ 37 static int 38 vtnet_netmap_reg(struct netmap_adapter *na, int state) 39 { 40 struct ifnet *ifp = na->ifp; 41 struct vtnet_softc *sc = ifp->if_softc; 42 43 /* Stop all txsync/rxsync and disable them. */ 44 netmap_disable_all_rings(ifp); 45 46 /* 47 * Trigger a device reinit, asking vtnet_init_locked() to 48 * also enter or exit netmap mode. 49 */ 50 VTNET_CORE_LOCK(sc); 51 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 52 vtnet_init_locked(sc, state ? VTNET_INIT_NETMAP_ENTER 53 : VTNET_INIT_NETMAP_EXIT); 54 VTNET_CORE_UNLOCK(sc); 55 56 /* Enable txsync/rxsync again. */ 57 netmap_enable_all_rings(ifp); 58 59 return (0); 60 } 61 62 63 /* Reconcile kernel and user view of the transmit ring. */ 64 static int 65 vtnet_netmap_txsync(struct netmap_kring *kring, int flags) 66 { 67 struct netmap_adapter *na = kring->na; 68 struct ifnet *ifp = na->ifp; 69 struct netmap_ring *ring = kring->ring; 70 u_int ring_nr = kring->ring_id; 71 u_int nm_i; /* index into the netmap ring */ 72 u_int const lim = kring->nkr_num_slots - 1; 73 u_int const head = kring->rhead; 74 75 /* device-specific */ 76 struct vtnet_softc *sc = ifp->if_softc; 77 struct vtnet_txq *txq = &sc->vtnet_txqs[ring_nr]; 78 struct virtqueue *vq = txq->vtntx_vq; 79 int interrupts = !(kring->nr_kflags & NKR_NOINTR); 80 u_int n; 81 82 /* 83 * First part: process new packets to send. 84 */ 85 86 nm_i = kring->nr_hwcur; 87 if (nm_i != head) { /* we have new packets to send */ 88 struct sglist *sg = txq->vtntx_sg; 89 90 for (; nm_i != head; nm_i = nm_next(nm_i, lim)) { 91 /* we use an empty header here */ 92 struct netmap_slot *slot = &ring->slot[nm_i]; 93 u_int len = slot->len; 94 uint64_t paddr; 95 void *addr = PNMB(na, slot, &paddr); 96 int err; 97 98 NM_CHECK_ADDR_LEN(na, addr, len); 99 100 slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); 101 /* Initialize the scatterlist, expose it to the hypervisor, 102 * and kick the hypervisor (if necessary). 103 */ 104 sglist_reset(sg); // cheap 105 err = sglist_append(sg, &txq->vtntx_shrhdr, sc->vtnet_hdr_size); 106 err |= sglist_append_phys(sg, paddr, len); 107 KASSERT(err == 0, ("%s: cannot append to sglist %d", 108 __func__, err)); 109 err = virtqueue_enqueue(vq, /*cookie=*/txq, sg, 110 /*readable=*/sg->sg_nseg, 111 /*writeable=*/0); 112 if (unlikely(err)) { 113 if (err != ENOSPC) 114 nm_prerr("virtqueue_enqueue(%s) failed: %d", 115 kring->name, err); 116 break; 117 } 118 } 119 120 virtqueue_notify(vq); 121 122 /* Update hwcur depending on where we stopped. */ 123 kring->nr_hwcur = nm_i; /* note we migth break early */ 124 } 125 126 /* Free used slots. We only consider our own used buffers, recognized 127 * by the token we passed to virtqueue_enqueue. 128 */ 129 n = 0; 130 for (;;) { 131 void *token = virtqueue_dequeue(vq, NULL); 132 if (token == NULL) 133 break; 134 if (unlikely(token != (void *)txq)) 135 nm_prerr("BUG: TX token mismatch"); 136 else 137 n++; 138 } 139 if (n > 0) { 140 kring->nr_hwtail += n; 141 if (kring->nr_hwtail > lim) 142 kring->nr_hwtail -= lim + 1; 143 } 144 145 if (interrupts && virtqueue_nfree(vq) < 32) 146 virtqueue_postpone_intr(vq, VQ_POSTPONE_LONG); 147 148 return 0; 149 } 150 151 /* 152 * Publish 'num 'netmap receive buffers to the host, starting 153 * from the next available one (rx->vtnrx_nm_refill). 154 * Return a positive error code on error, and 0 on success. 155 * If we could not publish all of the buffers that's an error, 156 * since the netmap ring and the virtqueue would go out of sync. 157 */ 158 static int 159 vtnet_netmap_kring_refill(struct netmap_kring *kring, u_int num) 160 { 161 struct netmap_adapter *na = kring->na; 162 struct ifnet *ifp = na->ifp; 163 struct netmap_ring *ring = kring->ring; 164 u_int ring_nr = kring->ring_id; 165 u_int const lim = kring->nkr_num_slots - 1; 166 u_int nm_i; 167 168 /* device-specific */ 169 struct vtnet_softc *sc = ifp->if_softc; 170 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[ring_nr]; 171 struct virtqueue *vq = rxq->vtnrx_vq; 172 173 /* use a local sglist, default might be short */ 174 struct sglist_seg ss[2]; 175 struct sglist sg = { ss, 0, 0, 2 }; 176 177 for (nm_i = rxq->vtnrx_nm_refill; num > 0; 178 nm_i = nm_next(nm_i, lim), num--) { 179 struct netmap_slot *slot = &ring->slot[nm_i]; 180 uint64_t paddr; 181 void *addr = PNMB(na, slot, &paddr); 182 int err; 183 184 if (addr == NETMAP_BUF_BASE(na)) { /* bad buf */ 185 if (netmap_ring_reinit(kring)) 186 return EFAULT; 187 } 188 189 slot->flags &= ~NS_BUF_CHANGED; 190 sglist_reset(&sg); 191 err = sglist_append(&sg, &rxq->vtnrx_shrhdr, sc->vtnet_hdr_size); 192 err |= sglist_append_phys(&sg, paddr, NETMAP_BUF_SIZE(na)); 193 KASSERT(err == 0, ("%s: cannot append to sglist %d", 194 __func__, err)); 195 /* writable for the host */ 196 err = virtqueue_enqueue(vq, /*cookie=*/rxq, &sg, 197 /*readable=*/0, /*writeable=*/sg.sg_nseg); 198 if (unlikely(err)) { 199 nm_prerr("virtqueue_enqueue(%s) failed: %d", 200 kring->name, err); 201 break; 202 } 203 } 204 rxq->vtnrx_nm_refill = nm_i; 205 206 return num == 0 ? 0 : ENOSPC; 207 } 208 209 /* 210 * Publish netmap buffers on a RX virtqueue. 211 * Returns -1 if this virtqueue is not being opened in netmap mode. 212 * If the virtqueue is being opened in netmap mode, return 0 on success and 213 * a positive error code on failure. 214 */ 215 static int 216 vtnet_netmap_rxq_populate(struct vtnet_rxq *rxq) 217 { 218 struct netmap_adapter *na = NA(rxq->vtnrx_sc->vtnet_ifp); 219 struct netmap_kring *kring; 220 struct netmap_slot *slot; 221 int error; 222 223 slot = netmap_reset(na, NR_RX, rxq->vtnrx_id, 0); 224 if (slot == NULL) 225 return -1; 226 kring = na->rx_rings[rxq->vtnrx_id]; 227 228 /* Expose all the RX netmap buffers we can. In case of no indirect 229 * buffers, the number of netmap slots in the RX ring matches the 230 * maximum number of 2-elements sglist that the RX virtqueue can 231 * accommodate. We need to start from kring->nr_hwcur, which is 0 232 * on netmap register and may be different from 0 if a virtio 233 * re-init happens while the device is in use by netmap. */ 234 rxq->vtnrx_nm_refill = kring->nr_hwcur; 235 error = vtnet_netmap_kring_refill(kring, na->num_rx_desc - 1); 236 virtqueue_notify(rxq->vtnrx_vq); 237 238 return error; 239 } 240 241 /* Reconcile kernel and user view of the receive ring. */ 242 static int 243 vtnet_netmap_rxsync(struct netmap_kring *kring, int flags) 244 { 245 struct netmap_adapter *na = kring->na; 246 struct ifnet *ifp = na->ifp; 247 struct netmap_ring *ring = kring->ring; 248 u_int ring_nr = kring->ring_id; 249 u_int nm_i; /* index into the netmap ring */ 250 u_int const lim = kring->nkr_num_slots - 1; 251 u_int const head = kring->rhead; 252 int force_update = (flags & NAF_FORCE_READ) || 253 (kring->nr_kflags & NKR_PENDINTR); 254 int interrupts = !(kring->nr_kflags & NKR_NOINTR); 255 256 /* device-specific */ 257 struct vtnet_softc *sc = ifp->if_softc; 258 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[ring_nr]; 259 struct virtqueue *vq = rxq->vtnrx_vq; 260 261 /* 262 * First part: import newly received packets. 263 * Only accept our own buffers (matching the token). We should only get 264 * matching buffers. The hwtail should never overrun hwcur, because 265 * we publish only N-1 receive buffers (and non N). 266 * In any case we must not leave this routine with the interrupts 267 * disabled, pending packets in the VQ and hwtail == (hwcur - 1), 268 * otherwise the pending packets could stall. 269 */ 270 if (netmap_no_pendintr || force_update) { 271 uint32_t hwtail_lim = nm_prev(kring->nr_hwcur, lim); 272 void *token; 273 274 vtnet_rxq_disable_intr(rxq); 275 276 nm_i = kring->nr_hwtail; 277 for (;;) { 278 int len; 279 token = virtqueue_dequeue(vq, &len); 280 if (token == NULL) { 281 /* 282 * Enable the interrupts again and double-check 283 * for more work. We can go on until we win the 284 * race condition, since we are not replenishing 285 * in the meanwhile, and thus we will process at 286 * most N-1 slots. 287 */ 288 if (interrupts && vtnet_rxq_enable_intr(rxq)) { 289 vtnet_rxq_disable_intr(rxq); 290 continue; 291 } 292 break; 293 } 294 if (unlikely(token != (void *)rxq)) { 295 nm_prerr("BUG: RX token mismatch"); 296 } else { 297 if (nm_i == hwtail_lim) { 298 KASSERT(false, ("hwtail would " 299 "overrun hwcur")); 300 } 301 302 /* Skip the virtio-net header. */ 303 len -= sc->vtnet_hdr_size; 304 if (unlikely(len < 0)) { 305 nm_prlim(1, "Truncated virtio-net-header, " 306 "missing %d bytes", -len); 307 len = 0; 308 } 309 ring->slot[nm_i].len = len; 310 ring->slot[nm_i].flags = 0; 311 nm_i = nm_next(nm_i, lim); 312 } 313 } 314 kring->nr_hwtail = nm_i; 315 kring->nr_kflags &= ~NKR_PENDINTR; 316 } 317 318 /* 319 * Second part: skip past packets that userspace has released. 320 */ 321 nm_i = kring->nr_hwcur; /* netmap ring index */ 322 if (nm_i != head) { 323 int released; 324 int error; 325 326 released = head - nm_i; 327 if (released < 0) 328 released += kring->nkr_num_slots; 329 error = vtnet_netmap_kring_refill(kring, released); 330 if (error) { 331 nm_prerr("Failed to replenish RX VQ with %u sgs", 332 released); 333 return error; 334 } 335 kring->nr_hwcur = head; 336 virtqueue_notify(vq); 337 } 338 339 nm_prdis("h %d c %d t %d hwcur %d hwtail %d", kring->rhead, 340 kring->rcur, kring->rtail, kring->nr_hwcur, kring->nr_hwtail); 341 342 return 0; 343 } 344 345 346 /* Enable/disable interrupts on all virtqueues. */ 347 static void 348 vtnet_netmap_intr(struct netmap_adapter *na, int state) 349 { 350 struct vtnet_softc *sc = na->ifp->if_softc; 351 int i; 352 353 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 354 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[i]; 355 struct vtnet_txq *txq = &sc->vtnet_txqs[i]; 356 struct virtqueue *txvq = txq->vtntx_vq; 357 358 if (state) { 359 vtnet_rxq_enable_intr(rxq); 360 virtqueue_enable_intr(txvq); 361 } else { 362 vtnet_rxq_disable_intr(rxq); 363 virtqueue_disable_intr(txvq); 364 } 365 } 366 } 367 368 static int 369 vtnet_netmap_tx_slots(struct vtnet_softc *sc) 370 { 371 int div; 372 373 /* We need to prepend a virtio-net header to each netmap buffer to be 374 * transmitted, therefore calling virtqueue_enqueue() passing sglist 375 * with 2 elements. 376 * TX virtqueues use indirect descriptors if the feature was negotiated 377 * with the host, and if sc->vtnet_tx_nsegs > 1. With indirect 378 * descriptors, a single virtio descriptor is sufficient to reference 379 * each TX sglist. Without them, we need two separate virtio descriptors 380 * for each TX sglist. We therefore compute the number of netmap TX 381 * slots according to these assumptions. 382 */ 383 if ((sc->vtnet_flags & VTNET_FLAG_INDIRECT) && sc->vtnet_tx_nsegs > 1) 384 div = 1; 385 else 386 div = 2; 387 388 return virtqueue_size(sc->vtnet_txqs[0].vtntx_vq) / div; 389 } 390 391 static int 392 vtnet_netmap_rx_slots(struct vtnet_softc *sc) 393 { 394 int div; 395 396 /* We need to prepend a virtio-net header to each netmap buffer to be 397 * received, therefore calling virtqueue_enqueue() passing sglist 398 * with 2 elements. 399 * RX virtqueues use indirect descriptors if the feature was negotiated 400 * with the host, and if sc->vtnet_rx_nsegs > 1. With indirect 401 * descriptors, a single virtio descriptor is sufficient to reference 402 * each RX sglist. Without them, we need two separate virtio descriptors 403 * for each RX sglist. We therefore compute the number of netmap RX 404 * slots according to these assumptions. 405 */ 406 if ((sc->vtnet_flags & VTNET_FLAG_INDIRECT) && sc->vtnet_rx_nsegs > 1) 407 div = 1; 408 else 409 div = 2; 410 411 return virtqueue_size(sc->vtnet_rxqs[0].vtnrx_vq) / div; 412 } 413 414 static int 415 vtnet_netmap_config(struct netmap_adapter *na, struct nm_config_info *info) 416 { 417 struct vtnet_softc *sc = na->ifp->if_softc; 418 419 info->num_tx_rings = sc->vtnet_act_vq_pairs; 420 info->num_rx_rings = sc->vtnet_act_vq_pairs; 421 info->num_tx_descs = vtnet_netmap_tx_slots(sc); 422 info->num_rx_descs = vtnet_netmap_rx_slots(sc); 423 info->rx_buf_maxsize = NETMAP_BUF_SIZE(na); 424 425 return 0; 426 } 427 428 static void 429 vtnet_netmap_attach(struct vtnet_softc *sc) 430 { 431 struct netmap_adapter na; 432 433 bzero(&na, sizeof(na)); 434 435 na.ifp = sc->vtnet_ifp; 436 na.na_flags = 0; 437 na.num_tx_desc = vtnet_netmap_tx_slots(sc); 438 na.num_rx_desc = vtnet_netmap_rx_slots(sc); 439 na.num_tx_rings = na.num_rx_rings = sc->vtnet_max_vq_pairs; 440 na.rx_buf_maxsize = 0; 441 na.nm_register = vtnet_netmap_reg; 442 na.nm_txsync = vtnet_netmap_txsync; 443 na.nm_rxsync = vtnet_netmap_rxsync; 444 na.nm_intr = vtnet_netmap_intr; 445 na.nm_config = vtnet_netmap_config; 446 447 netmap_attach(&na); 448 449 nm_prinf("vtnet attached txq=%d, txd=%d rxq=%d, rxd=%d", 450 na.num_tx_rings, na.num_tx_desc, 451 na.num_tx_rings, na.num_rx_desc); 452 } 453 /* end of file */ 454