1 /* 2 * Copyright (C) 2014 Vincenzo Maffione, Luigi Rizzo. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 /* 27 * $FreeBSD$ 28 */ 29 30 #include <net/netmap.h> 31 #include <sys/selinfo.h> 32 #include <vm/vm.h> 33 #include <vm/pmap.h> /* vtophys ? */ 34 #include <dev/netmap/netmap_kern.h> 35 36 37 #define SOFTC_T vtnet_softc 38 39 /* Free all the unused buffer in all the RX virtqueues. 40 * This function is called when entering and exiting netmap mode. 41 * - buffers queued by the virtio driver return skbuf/mbuf pointer 42 * and need to be freed; 43 * - buffers queued by netmap return the txq/rxq, and do not need work 44 */ 45 static void 46 vtnet_netmap_free_bufs(struct SOFTC_T* sc) 47 { 48 int i, nmb = 0, n = 0, last; 49 50 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 51 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[i]; 52 struct virtqueue *vq; 53 struct mbuf *m; 54 struct vtnet_txq *txq = &sc->vtnet_txqs[i]; 55 struct vtnet_tx_header *txhdr; 56 57 last = 0; 58 vq = rxq->vtnrx_vq; 59 while ((m = virtqueue_drain(vq, &last)) != NULL) { 60 n++; 61 if (m != (void *)rxq) 62 m_freem(m); 63 else 64 nmb++; 65 } 66 67 last = 0; 68 vq = txq->vtntx_vq; 69 while ((txhdr = virtqueue_drain(vq, &last)) != NULL) { 70 n++; 71 if (txhdr != (void *)txq) { 72 m_freem(txhdr->vth_mbuf); 73 uma_zfree(vtnet_tx_header_zone, txhdr); 74 } else 75 nmb++; 76 } 77 } 78 D("freed %d mbufs, %d netmap bufs on %d queues", 79 n - nmb, nmb, i); 80 } 81 82 /* Register and unregister. */ 83 static int 84 vtnet_netmap_reg(struct netmap_adapter *na, int onoff) 85 { 86 struct ifnet *ifp = na->ifp; 87 struct SOFTC_T *sc = ifp->if_softc; 88 89 VTNET_CORE_LOCK(sc); 90 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 91 /* enable or disable flags and callbacks in na and ifp */ 92 if (onoff) { 93 nm_set_native_flags(na); 94 } else { 95 nm_clear_native_flags(na); 96 } 97 /* drain queues so netmap and native drivers 98 * do not interfere with each other 99 */ 100 vtnet_netmap_free_bufs(sc); 101 vtnet_init_locked(sc); /* also enable intr */ 102 VTNET_CORE_UNLOCK(sc); 103 return (ifp->if_drv_flags & IFF_DRV_RUNNING ? 0 : 1); 104 } 105 106 107 /* Reconcile kernel and user view of the transmit ring. */ 108 static int 109 vtnet_netmap_txsync(struct netmap_kring *kring, int flags) 110 { 111 struct netmap_adapter *na = kring->na; 112 struct ifnet *ifp = na->ifp; 113 struct netmap_ring *ring = kring->ring; 114 u_int ring_nr = kring->ring_id; 115 u_int nm_i; /* index into the netmap ring */ 116 u_int nic_i; /* index into the NIC ring */ 117 u_int n; 118 u_int const lim = kring->nkr_num_slots - 1; 119 u_int const head = kring->rhead; 120 121 /* device-specific */ 122 struct SOFTC_T *sc = ifp->if_softc; 123 struct vtnet_txq *txq = &sc->vtnet_txqs[ring_nr]; 124 struct virtqueue *vq = txq->vtntx_vq; 125 126 /* 127 * First part: process new packets to send. 128 */ 129 rmb(); 130 131 nm_i = kring->nr_hwcur; 132 if (nm_i != head) { /* we have new packets to send */ 133 struct sglist *sg = txq->vtntx_sg; 134 135 nic_i = netmap_idx_k2n(kring, nm_i); 136 for (n = 0; nm_i != head; n++) { 137 /* we use an empty header here */ 138 static struct virtio_net_hdr_mrg_rxbuf hdr; 139 struct netmap_slot *slot = &ring->slot[nm_i]; 140 u_int len = slot->len; 141 uint64_t paddr; 142 void *addr = PNMB(na, slot, &paddr); 143 int err; 144 145 NM_CHECK_ADDR_LEN(na, addr, len); 146 147 slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); 148 /* Initialize the scatterlist, expose it to the hypervisor, 149 * and kick the hypervisor (if necessary). 150 */ 151 sglist_reset(sg); // cheap 152 // if vtnet_hdr_size > 0 ... 153 err = sglist_append(sg, &hdr, sc->vtnet_hdr_size); 154 // XXX later, support multi segment 155 err = sglist_append_phys(sg, paddr, len); 156 /* use na as the cookie */ 157 err = virtqueue_enqueue(vq, txq, sg, sg->sg_nseg, 0); 158 if (unlikely(err < 0)) { 159 D("virtqueue_enqueue failed"); 160 break; 161 } 162 163 nm_i = nm_next(nm_i, lim); 164 nic_i = nm_next(nic_i, lim); 165 } 166 /* Update hwcur depending on where we stopped. */ 167 kring->nr_hwcur = nm_i; /* note we migth break early */ 168 169 /* No more free TX slots? Ask the hypervisor for notifications, 170 * possibly only when a considerable amount of work has been 171 * done. 172 */ 173 ND(3,"sent %d packets, hwcur %d", n, nm_i); 174 virtqueue_disable_intr(vq); 175 virtqueue_notify(vq); 176 } else { 177 if (ring->head != ring->tail) 178 ND(5, "pure notify ? head %d tail %d nused %d %d", 179 ring->head, ring->tail, virtqueue_nused(vq), 180 (virtqueue_dump(vq), 1)); 181 virtqueue_notify(vq); 182 virtqueue_enable_intr(vq); // like postpone with 0 183 } 184 185 186 /* Free used slots. We only consider our own used buffers, recognized 187 * by the token we passed to virtqueue_add_outbuf. 188 */ 189 n = 0; 190 for (;;) { 191 struct vtnet_tx_header *txhdr = virtqueue_dequeue(vq, NULL); 192 if (txhdr == NULL) 193 break; 194 if (likely(txhdr == (void *)txq)) { 195 n++; 196 if (virtqueue_nused(vq) < 32) { // XXX slow release 197 break; 198 } 199 } else { /* leftover from previous transmission */ 200 m_freem(txhdr->vth_mbuf); 201 uma_zfree(vtnet_tx_header_zone, txhdr); 202 } 203 } 204 if (n) { 205 kring->nr_hwtail += n; 206 if (kring->nr_hwtail > lim) 207 kring->nr_hwtail -= lim + 1; 208 } 209 if (nm_i != kring->nr_hwtail /* && vtnet_txq_below_threshold(txq) == 0*/) { 210 ND(3, "disable intr, hwcur %d", nm_i); 211 virtqueue_disable_intr(vq); 212 } else { 213 ND(3, "enable intr, hwcur %d", nm_i); 214 virtqueue_postpone_intr(vq, VQ_POSTPONE_SHORT); 215 } 216 217 return 0; 218 } 219 220 static int 221 vtnet_refill_rxq(struct netmap_kring *kring, u_int nm_i, u_int head) 222 { 223 struct netmap_adapter *na = kring->na; 224 struct ifnet *ifp = na->ifp; 225 struct netmap_ring *ring = kring->ring; 226 u_int ring_nr = kring->ring_id; 227 u_int const lim = kring->nkr_num_slots - 1; 228 u_int n; 229 230 /* device-specific */ 231 struct SOFTC_T *sc = ifp->if_softc; 232 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[ring_nr]; 233 struct virtqueue *vq = rxq->vtnrx_vq; 234 235 /* use a local sglist, default might be short */ 236 struct sglist_seg ss[2]; 237 struct sglist sg = { ss, 0, 0, 2 }; 238 239 for (n = 0; nm_i != head; n++) { 240 static struct virtio_net_hdr_mrg_rxbuf hdr; 241 struct netmap_slot *slot = &ring->slot[nm_i]; 242 uint64_t paddr; 243 void *addr = PNMB(na, slot, &paddr); 244 int err = 0; 245 246 if (addr == NETMAP_BUF_BASE(na)) { /* bad buf */ 247 if (netmap_ring_reinit(kring)) 248 return -1; 249 } 250 251 slot->flags &= ~NS_BUF_CHANGED; 252 sglist_reset(&sg); // cheap 253 err = sglist_append(&sg, &hdr, sc->vtnet_hdr_size); 254 err = sglist_append_phys(&sg, paddr, NETMAP_BUF_SIZE(na)); 255 /* writable for the host */ 256 err = virtqueue_enqueue(vq, rxq, &sg, 0, sg.sg_nseg); 257 if (err < 0) { 258 D("virtqueue_enqueue failed"); 259 break; 260 } 261 nm_i = nm_next(nm_i, lim); 262 } 263 return nm_i; 264 } 265 266 /* Reconcile kernel and user view of the receive ring. */ 267 static int 268 vtnet_netmap_rxsync(struct netmap_kring *kring, int flags) 269 { 270 struct netmap_adapter *na = kring->na; 271 struct ifnet *ifp = na->ifp; 272 struct netmap_ring *ring = kring->ring; 273 u_int ring_nr = kring->ring_id; 274 u_int nm_i; /* index into the netmap ring */ 275 // u_int nic_i; /* index into the NIC ring */ 276 u_int n; 277 u_int const lim = kring->nkr_num_slots - 1; 278 u_int const head = kring->rhead; 279 int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR; 280 281 /* device-specific */ 282 struct SOFTC_T *sc = ifp->if_softc; 283 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[ring_nr]; 284 struct virtqueue *vq = rxq->vtnrx_vq; 285 286 /* XXX netif_carrier_ok ? */ 287 288 if (head > lim) 289 return netmap_ring_reinit(kring); 290 291 rmb(); 292 /* 293 * First part: import newly received packets. 294 * Only accept our 295 * own buffers (matching the token). We should only get 296 * matching buffers, because of vtnet_netmap_free_rx_unused_bufs() 297 * and vtnet_netmap_init_buffers(). 298 */ 299 if (netmap_no_pendintr || force_update) { 300 uint16_t slot_flags = kring->nkr_slot_flags; 301 struct netmap_adapter *token; 302 303 nm_i = kring->nr_hwtail; 304 n = 0; 305 for (;;) { 306 int len; 307 token = virtqueue_dequeue(vq, &len); 308 if (token == NULL) 309 break; 310 if (likely(token == (void *)rxq)) { 311 ring->slot[nm_i].len = len; 312 ring->slot[nm_i].flags = slot_flags; 313 nm_i = nm_next(nm_i, lim); 314 n++; 315 } else { 316 D("This should not happen"); 317 } 318 } 319 kring->nr_hwtail = nm_i; 320 kring->nr_kflags &= ~NKR_PENDINTR; 321 } 322 ND("[B] h %d c %d hwcur %d hwtail %d", 323 ring->head, ring->cur, kring->nr_hwcur, 324 kring->nr_hwtail); 325 326 /* 327 * Second part: skip past packets that userspace has released. 328 */ 329 nm_i = kring->nr_hwcur; /* netmap ring index */ 330 if (nm_i != head) { 331 int err = vtnet_refill_rxq(kring, nm_i, head); 332 if (err < 0) 333 return 1; 334 kring->nr_hwcur = err; 335 virtqueue_notify(vq); 336 /* After draining the queue may need an intr from the hypervisor */ 337 vtnet_rxq_enable_intr(rxq); 338 } 339 340 ND("[C] h %d c %d t %d hwcur %d hwtail %d", 341 ring->head, ring->cur, ring->tail, 342 kring->nr_hwcur, kring->nr_hwtail); 343 344 return 0; 345 } 346 347 348 /* Make RX virtqueues buffers pointing to netmap buffers. */ 349 static int 350 vtnet_netmap_init_rx_buffers(struct SOFTC_T *sc) 351 { 352 struct ifnet *ifp = sc->vtnet_ifp; 353 struct netmap_adapter* na = NA(ifp); 354 unsigned int r; 355 356 if (!nm_native_on(na)) 357 return 0; 358 for (r = 0; r < na->num_rx_rings; r++) { 359 struct netmap_kring *kring = &na->rx_rings[r]; 360 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[r]; 361 struct virtqueue *vq = rxq->vtnrx_vq; 362 struct netmap_slot* slot; 363 int err = 0; 364 365 slot = netmap_reset(na, NR_RX, r, 0); 366 if (!slot) { 367 D("strange, null netmap ring %d", r); 368 return 0; 369 } 370 /* Add up to na>-num_rx_desc-1 buffers to this RX virtqueue. 371 * It's important to leave one virtqueue slot free, otherwise 372 * we can run into ring->cur/ring->tail wraparounds. 373 */ 374 err = vtnet_refill_rxq(kring, 0, na->num_rx_desc-1); 375 if (err < 0) 376 return 0; 377 virtqueue_notify(vq); 378 } 379 380 return 1; 381 } 382 383 /* Update the virtio-net device configurations. Number of queues can 384 * change dinamically, by 'ethtool --set-channels $IFNAME combined $N'. 385 * This is actually the only way virtio-net can currently enable 386 * the multiqueue mode. 387 * XXX note that we seem to lose packets if the netmap ring has more 388 * slots than the queue 389 */ 390 static int 391 vtnet_netmap_config(struct netmap_adapter *na, u_int *txr, u_int *txd, 392 u_int *rxr, u_int *rxd) 393 { 394 struct ifnet *ifp = na->ifp; 395 struct SOFTC_T *sc = ifp->if_softc; 396 397 *txr = *rxr = sc->vtnet_max_vq_pairs; 398 *rxd = 512; // sc->vtnet_rx_nmbufs; 399 *txd = *rxd; // XXX 400 D("vtnet config txq=%d, txd=%d rxq=%d, rxd=%d", 401 *txr, *txd, *rxr, *rxd); 402 403 return 0; 404 } 405 406 static void 407 vtnet_netmap_attach(struct SOFTC_T *sc) 408 { 409 struct netmap_adapter na; 410 411 bzero(&na, sizeof(na)); 412 413 na.ifp = sc->vtnet_ifp; 414 na.num_tx_desc = 1024;// sc->vtnet_rx_nmbufs; 415 na.num_rx_desc = 1024; // sc->vtnet_rx_nmbufs; 416 na.nm_register = vtnet_netmap_reg; 417 na.nm_txsync = vtnet_netmap_txsync; 418 na.nm_rxsync = vtnet_netmap_rxsync; 419 na.nm_config = vtnet_netmap_config; 420 na.num_tx_rings = na.num_rx_rings = sc->vtnet_max_vq_pairs; 421 D("max rings %d", sc->vtnet_max_vq_pairs); 422 netmap_attach(&na); 423 424 D("virtio attached txq=%d, txd=%d rxq=%d, rxd=%d", 425 na.num_tx_rings, na.num_tx_desc, 426 na.num_tx_rings, na.num_rx_desc); 427 } 428 /* end of file */ 429