xref: /freebsd/sys/dev/netmap/if_vtnet_netmap.h (revision 3823d5e198425b4f5e5a80267d195769d1063773)
1 /*
2  * Copyright (C) 2014 Vincenzo Maffione, Luigi Rizzo. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23  * SUCH DAMAGE.
24  */
25 
26 /*
27  * $FreeBSD$
28  */
29 
30 #include <net/netmap.h>
31 #include <sys/selinfo.h>
32 #include <vm/vm.h>
33 #include <vm/pmap.h>    /* vtophys ? */
34 #include <dev/netmap/netmap_kern.h>
35 
36 
37 #define SOFTC_T	vtnet_softc
38 
39 /* Free all the unused buffer in all the RX virtqueues.
40  * This function is called when entering and exiting netmap mode.
41  * - buffers queued by the virtio driver return skbuf/mbuf pointer
42  *   and need to be freed;
43  * - buffers queued by netmap return the txq/rxq, and do not need work
44  */
45 static void
46 vtnet_netmap_free_bufs(struct SOFTC_T* sc)
47 {
48 	int i, nmb = 0, n = 0, last;
49 
50 	for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
51 		struct vtnet_rxq *rxq = &sc->vtnet_rxqs[i];
52 		struct virtqueue *vq;
53 		struct mbuf *m;
54 		struct vtnet_txq *txq = &sc->vtnet_txqs[i];
55                 struct vtnet_tx_header *txhdr;
56 
57 		last = 0;
58 		vq = rxq->vtnrx_vq;
59 		while ((m = virtqueue_drain(vq, &last)) != NULL) {
60 			n++;
61 			if (m != (void *)rxq)
62 				m_freem(m);
63 			else
64 				nmb++;
65 		}
66 
67 		last = 0;
68 		vq = txq->vtntx_vq;
69 		while ((txhdr = virtqueue_drain(vq, &last)) != NULL) {
70 			n++;
71 			if (txhdr != (void *)txq) {
72 				m_freem(txhdr->vth_mbuf);
73 				uma_zfree(vtnet_tx_header_zone, txhdr);
74 			} else
75 				nmb++;
76 		}
77 	}
78 	D("freed %d mbufs, %d netmap bufs on %d queues",
79 		n - nmb, nmb, i);
80 }
81 
82 /* Register and unregister. */
83 static int
84 vtnet_netmap_reg(struct netmap_adapter *na, int onoff)
85 {
86         struct ifnet *ifp = na->ifp;
87 	struct SOFTC_T *sc = ifp->if_softc;
88 
89 	VTNET_CORE_LOCK(sc);
90 	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
91 	/* enable or disable flags and callbacks in na and ifp */
92 	if (onoff) {
93 		nm_set_native_flags(na);
94 	} else {
95 		nm_clear_native_flags(na);
96 	}
97 	/* drain queues so netmap and native drivers
98 	 * do not interfere with each other
99 	 */
100 	vtnet_netmap_free_bufs(sc);
101         vtnet_init_locked(sc);       /* also enable intr */
102         VTNET_CORE_UNLOCK(sc);
103         return (ifp->if_drv_flags & IFF_DRV_RUNNING ? 0 : 1);
104 }
105 
106 
107 /* Reconcile kernel and user view of the transmit ring. */
108 static int
109 vtnet_netmap_txsync(struct netmap_kring *kring, int flags)
110 {
111 	struct netmap_adapter *na = kring->na;
112         struct ifnet *ifp = na->ifp;
113 	struct netmap_ring *ring = kring->ring;
114 	u_int ring_nr = kring->ring_id;
115 	u_int nm_i;	/* index into the netmap ring */
116 	u_int nic_i;	/* index into the NIC ring */
117 	u_int n;
118 	u_int const lim = kring->nkr_num_slots - 1;
119 	u_int const head = kring->rhead;
120 
121 	/* device-specific */
122 	struct SOFTC_T *sc = ifp->if_softc;
123 	struct vtnet_txq *txq = &sc->vtnet_txqs[ring_nr];
124 	struct virtqueue *vq = txq->vtntx_vq;
125 
126 	/*
127 	 * First part: process new packets to send.
128 	 */
129 	rmb();
130 
131 	nm_i = kring->nr_hwcur;
132 	if (nm_i != head) {	/* we have new packets to send */
133 		struct sglist *sg = txq->vtntx_sg;
134 
135 		nic_i = netmap_idx_k2n(kring, nm_i);
136 		for (n = 0; nm_i != head; n++) {
137 			/* we use an empty header here */
138 			static struct virtio_net_hdr_mrg_rxbuf hdr;
139 			struct netmap_slot *slot = &ring->slot[nm_i];
140 			u_int len = slot->len;
141 			uint64_t paddr;
142 			void *addr = PNMB(na, slot, &paddr);
143                         int err;
144 
145 			NM_CHECK_ADDR_LEN(na, addr, len);
146 
147 			slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED);
148 			/* Initialize the scatterlist, expose it to the hypervisor,
149 			 * and kick the hypervisor (if necessary).
150 			 */
151 			sglist_reset(sg); // cheap
152 			// if vtnet_hdr_size > 0 ...
153 			err = sglist_append(sg, &hdr, sc->vtnet_hdr_size);
154 			// XXX later, support multi segment
155 			err = sglist_append_phys(sg, paddr, len);
156 			/* use na as the cookie */
157                         err = virtqueue_enqueue(vq, txq, sg, sg->sg_nseg, 0);
158                         if (unlikely(err < 0)) {
159                                 D("virtqueue_enqueue failed");
160                                 break;
161                         }
162 
163 			nm_i = nm_next(nm_i, lim);
164 			nic_i = nm_next(nic_i, lim);
165 		}
166 		/* Update hwcur depending on where we stopped. */
167 		kring->nr_hwcur = nm_i; /* note we migth break early */
168 
169 		/* No more free TX slots? Ask the hypervisor for notifications,
170 		 * possibly only when a considerable amount of work has been
171 		 * done.
172 		 */
173 		ND(3,"sent %d packets, hwcur %d", n, nm_i);
174 		virtqueue_disable_intr(vq);
175 		virtqueue_notify(vq);
176 	} else {
177 		if (ring->head != ring->tail)
178 		    ND(5, "pure notify ? head %d tail %d nused %d %d",
179 			ring->head, ring->tail, virtqueue_nused(vq),
180 			(virtqueue_dump(vq), 1));
181 		virtqueue_notify(vq);
182 		virtqueue_enable_intr(vq); // like postpone with 0
183 	}
184 
185 
186         /* Free used slots. We only consider our own used buffers, recognized
187 	 * by the token we passed to virtqueue_add_outbuf.
188 	 */
189         n = 0;
190         for (;;) {
191                 struct vtnet_tx_header *txhdr = virtqueue_dequeue(vq, NULL);
192                 if (txhdr == NULL)
193                         break;
194                 if (likely(txhdr == (void *)txq)) {
195                         n++;
196 			if (virtqueue_nused(vq) < 32) { // XXX slow release
197 				break;
198 			}
199 		} else { /* leftover from previous transmission */
200 			m_freem(txhdr->vth_mbuf);
201 			uma_zfree(vtnet_tx_header_zone, txhdr);
202 		}
203         }
204 	if (n) {
205 		kring->nr_hwtail += n;
206 		if (kring->nr_hwtail > lim)
207 			kring->nr_hwtail -= lim + 1;
208 	}
209 	if (nm_i != kring->nr_hwtail /* && vtnet_txq_below_threshold(txq) == 0*/) {
210 		ND(3, "disable intr, hwcur %d", nm_i);
211 		virtqueue_disable_intr(vq);
212 	} else {
213 		ND(3, "enable intr, hwcur %d", nm_i);
214 		virtqueue_postpone_intr(vq, VQ_POSTPONE_SHORT);
215 	}
216 
217 //out:
218 	nm_txsync_finalize(kring);
219 
220         return 0;
221 }
222 
223 static int
224 vtnet_refill_rxq(struct netmap_kring *kring, u_int nm_i, u_int head)
225 {
226 	struct netmap_adapter *na = kring->na;
227         struct ifnet *ifp = na->ifp;
228 	struct netmap_ring *ring = kring->ring;
229 	u_int ring_nr = kring->ring_id;
230 	u_int const lim = kring->nkr_num_slots - 1;
231 	u_int n;
232 
233 	/* device-specific */
234 	struct SOFTC_T *sc = ifp->if_softc;
235 	struct vtnet_rxq *rxq = &sc->vtnet_rxqs[ring_nr];
236 	struct virtqueue *vq = rxq->vtnrx_vq;
237 
238 	/* use a local sglist, default might be short */
239 	struct sglist_seg ss[2];
240 	struct sglist sg = { ss, 0, 0, 2 };
241 
242 	for (n = 0; nm_i != head; n++) {
243 		static struct virtio_net_hdr_mrg_rxbuf hdr;
244 		struct netmap_slot *slot = &ring->slot[nm_i];
245 		uint64_t paddr;
246 		void *addr = PNMB(na, slot, &paddr);
247 		int err = 0;
248 
249 		if (addr == NETMAP_BUF_BASE(na)) { /* bad buf */
250 			if (netmap_ring_reinit(kring))
251 				return -1;
252 		}
253 
254 		slot->flags &= ~NS_BUF_CHANGED;
255 		sglist_reset(&sg); // cheap
256 		err = sglist_append(&sg, &hdr, sc->vtnet_hdr_size);
257 		err = sglist_append_phys(&sg, paddr, NETMAP_BUF_SIZE(na));
258 		/* writable for the host */
259 		err = virtqueue_enqueue(vq, rxq, &sg, 0, sg.sg_nseg);
260 		if (err < 0) {
261 			D("virtqueue_enqueue failed");
262 			break;
263 		}
264 		nm_i = nm_next(nm_i, lim);
265 	}
266 	return nm_i;
267 }
268 
269 /* Reconcile kernel and user view of the receive ring. */
270 static int
271 vtnet_netmap_rxsync(struct netmap_kring *kring, int flags)
272 {
273 	struct netmap_adapter *na = kring->na;
274         struct ifnet *ifp = na->ifp;
275 	struct netmap_ring *ring = kring->ring;
276 	u_int ring_nr = kring->ring_id;
277 	u_int nm_i;	/* index into the netmap ring */
278 	// u_int nic_i;	/* index into the NIC ring */
279 	u_int n;
280 	u_int const lim = kring->nkr_num_slots - 1;
281 	u_int const head = nm_rxsync_prologue(kring);
282 	int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;
283 
284 	/* device-specific */
285 	struct SOFTC_T *sc = ifp->if_softc;
286 	struct vtnet_rxq *rxq = &sc->vtnet_rxqs[ring_nr];
287 	struct virtqueue *vq = rxq->vtnrx_vq;
288 
289 	/* XXX netif_carrier_ok ? */
290 
291 	if (head > lim)
292 		return netmap_ring_reinit(kring);
293 
294 	rmb();
295 	/*
296 	 * First part: import newly received packets.
297 	 * Only accept our
298 	 * own buffers (matching the token). We should only get
299 	 * matching buffers, because of vtnet_netmap_free_rx_unused_bufs()
300 	 * and vtnet_netmap_init_buffers().
301 	 */
302 	if (netmap_no_pendintr || force_update) {
303 		uint16_t slot_flags = kring->nkr_slot_flags;
304                 struct netmap_adapter *token;
305 
306                 nm_i = kring->nr_hwtail;
307                 n = 0;
308 		for (;;) {
309 			int len;
310                         token = virtqueue_dequeue(vq, &len);
311                         if (token == NULL)
312                                 break;
313                         if (likely(token == (void *)rxq)) {
314                             ring->slot[nm_i].len = len;
315                             ring->slot[nm_i].flags = slot_flags;
316                             nm_i = nm_next(nm_i, lim);
317                             n++;
318                         } else {
319 			    D("This should not happen");
320                         }
321 		}
322 		kring->nr_hwtail = nm_i;
323 		kring->nr_kflags &= ~NKR_PENDINTR;
324 	}
325         ND("[B] h %d c %d hwcur %d hwtail %d",
326 		ring->head, ring->cur, kring->nr_hwcur,
327 			      kring->nr_hwtail);
328 
329 	/*
330 	 * Second part: skip past packets that userspace has released.
331 	 */
332 	nm_i = kring->nr_hwcur; /* netmap ring index */
333 	if (nm_i != head) {
334 		int err = vtnet_refill_rxq(kring, nm_i, head);
335 		if (err < 0)
336 			return 1;
337 		kring->nr_hwcur = err;
338 		virtqueue_notify(vq);
339 		/* After draining the queue may need an intr from the hypervisor */
340         	vtnet_rxq_enable_intr(rxq);
341 	}
342 
343 	/* tell userspace that there might be new packets. */
344 	nm_rxsync_finalize(kring);
345 
346         ND("[C] h %d c %d t %d hwcur %d hwtail %d",
347 		ring->head, ring->cur, ring->tail,
348 		kring->nr_hwcur, kring->nr_hwtail);
349 
350 	return 0;
351 }
352 
353 
354 /* Make RX virtqueues buffers pointing to netmap buffers. */
355 static int
356 vtnet_netmap_init_rx_buffers(struct SOFTC_T *sc)
357 {
358 	struct ifnet *ifp = sc->vtnet_ifp;
359 	struct netmap_adapter* na = NA(ifp);
360 	unsigned int r;
361 
362 	if (!nm_native_on(na))
363 		return 0;
364 	for (r = 0; r < na->num_rx_rings; r++) {
365                 struct netmap_kring *kring = &na->rx_rings[r];
366 		struct vtnet_rxq *rxq = &sc->vtnet_rxqs[r];
367 		struct virtqueue *vq = rxq->vtnrx_vq;
368 	        struct netmap_slot* slot;
369 		int err = 0;
370 
371 		slot = netmap_reset(na, NR_RX, r, 0);
372 		if (!slot) {
373 			D("strange, null netmap ring %d", r);
374 			return 0;
375 		}
376 		/* Add up to na>-num_rx_desc-1 buffers to this RX virtqueue.
377 		 * It's important to leave one virtqueue slot free, otherwise
378 		 * we can run into ring->cur/ring->tail wraparounds.
379 		 */
380 		err = vtnet_refill_rxq(kring, 0, na->num_rx_desc-1);
381 		if (err < 0)
382 			return 0;
383 		virtqueue_notify(vq);
384 	}
385 
386 	return 1;
387 }
388 
389 /* Update the virtio-net device configurations. Number of queues can
390  * change dinamically, by 'ethtool --set-channels $IFNAME combined $N'.
391  * This is actually the only way virtio-net can currently enable
392  * the multiqueue mode.
393  * XXX note that we seem to lose packets if the netmap ring has more
394  * slots than the queue
395  */
396 static int
397 vtnet_netmap_config(struct netmap_adapter *na, u_int *txr, u_int *txd,
398 						u_int *rxr, u_int *rxd)
399 {
400 	struct ifnet *ifp = na->ifp;
401 	struct SOFTC_T *sc = ifp->if_softc;
402 
403 	*txr = *rxr = sc->vtnet_max_vq_pairs;
404 	*rxd = 512; // sc->vtnet_rx_nmbufs;
405 	*txd = *rxd; // XXX
406         D("vtnet config txq=%d, txd=%d rxq=%d, rxd=%d",
407 					*txr, *txd, *rxr, *rxd);
408 
409 	return 0;
410 }
411 
412 static void
413 vtnet_netmap_attach(struct SOFTC_T *sc)
414 {
415 	struct netmap_adapter na;
416 
417 	bzero(&na, sizeof(na));
418 
419 	na.ifp = sc->vtnet_ifp;
420 	na.num_tx_desc =  1024;// sc->vtnet_rx_nmbufs;
421 	na.num_rx_desc =  1024; // sc->vtnet_rx_nmbufs;
422 	na.nm_register = vtnet_netmap_reg;
423 	na.nm_txsync = vtnet_netmap_txsync;
424 	na.nm_rxsync = vtnet_netmap_rxsync;
425 	na.nm_config = vtnet_netmap_config;
426 	na.num_tx_rings = na.num_rx_rings = sc->vtnet_max_vq_pairs;
427 	D("max rings %d", sc->vtnet_max_vq_pairs);
428 	netmap_attach(&na);
429 
430         D("virtio attached txq=%d, txd=%d rxq=%d, rxd=%d",
431 			na.num_tx_rings, na.num_tx_desc,
432 			na.num_tx_rings, na.num_rx_desc);
433 }
434 /* end of file */
435