xref: /freebsd/sys/dev/xen/netfront/netfront.c (revision 9bc300465e48e19d794d88d0c158a2adb92c7197)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2004-2006 Kip Macy
5  * Copyright (c) 2015 Wei Liu <wei.liu2@citrix.com>
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include <sys/cdefs.h>
31 #include "opt_inet.h"
32 #include "opt_inet6.h"
33 
34 #include <sys/param.h>
35 #include <sys/sockio.h>
36 #include <sys/limits.h>
37 #include <sys/mbuf.h>
38 #include <sys/malloc.h>
39 #include <sys/module.h>
40 #include <sys/kernel.h>
41 #include <sys/socket.h>
42 #include <sys/sysctl.h>
43 #include <sys/taskqueue.h>
44 
45 #include <net/if.h>
46 #include <net/if_var.h>
47 #include <net/if_arp.h>
48 #include <net/ethernet.h>
49 #include <net/if_media.h>
50 #include <net/bpf.h>
51 #include <net/if_types.h>
52 
53 #include <netinet/in.h>
54 #include <netinet/ip.h>
55 #include <netinet/if_ether.h>
56 #include <netinet/tcp.h>
57 #include <netinet/tcp_lro.h>
58 
59 #include <vm/vm.h>
60 #include <vm/pmap.h>
61 
62 #include <sys/bus.h>
63 
64 #include <xen/xen-os.h>
65 #include <xen/hypervisor.h>
66 #include <xen/xen_intr.h>
67 #include <xen/gnttab.h>
68 #include <contrib/xen/memory.h>
69 #include <contrib/xen/io/netif.h>
70 #include <xen/xenbus/xenbusvar.h>
71 
72 #include <machine/bus.h>
73 
74 #include "xenbus_if.h"
75 
76 /* Features supported by all backends.  TSO and LRO can be negotiated */
77 #define XN_CSUM_FEATURES	(CSUM_TCP | CSUM_UDP)
78 
79 #define NET_TX_RING_SIZE __CONST_RING_SIZE(netif_tx, PAGE_SIZE)
80 #define NET_RX_RING_SIZE __CONST_RING_SIZE(netif_rx, PAGE_SIZE)
81 
82 #define NET_RX_SLOTS_MIN (XEN_NETIF_NR_SLOTS_MIN + 1)
83 
84 /*
85  * Should the driver do LRO on the RX end
86  *  this can be toggled on the fly, but the
87  *  interface must be reset (down/up) for it
88  *  to take effect.
89  */
90 static int xn_enable_lro = 1;
91 TUNABLE_INT("hw.xn.enable_lro", &xn_enable_lro);
92 
93 /*
94  * Number of pairs of queues.
95  */
96 static unsigned long xn_num_queues = 4;
97 TUNABLE_ULONG("hw.xn.num_queues", &xn_num_queues);
98 
99 /**
100  * \brief The maximum allowed data fragments in a single transmit
101  *        request.
102  *
103  * This limit is imposed by the backend driver.  We assume here that
104  * we are dealing with a Linux driver domain and have set our limit
105  * to mirror the Linux MAX_SKB_FRAGS constant.
106  */
107 #define	MAX_TX_REQ_FRAGS (65536 / PAGE_SIZE + 2)
108 
109 #define RX_COPY_THRESHOLD 256
110 
111 #define net_ratelimit() 0
112 
113 struct netfront_rxq;
114 struct netfront_txq;
115 struct netfront_info;
116 struct netfront_rx_info;
117 
118 static void xn_txeof(struct netfront_txq *);
119 static void xn_rxeof(struct netfront_rxq *);
120 static void xn_alloc_rx_buffers(struct netfront_rxq *);
121 static void xn_alloc_rx_buffers_callout(void *arg);
122 
123 static void xn_release_rx_bufs(struct netfront_rxq *);
124 static void xn_release_tx_bufs(struct netfront_txq *);
125 
126 static void xn_rxq_intr(struct netfront_rxq *);
127 static void xn_txq_intr(struct netfront_txq *);
128 static void xn_intr(void *);
129 static int xn_assemble_tx_request(struct netfront_txq *, struct mbuf *);
130 static int xn_ioctl(if_t, u_long, caddr_t);
131 static void xn_ifinit_locked(struct netfront_info *);
132 static void xn_ifinit(void *);
133 static void xn_stop(struct netfront_info *);
134 static void xn_query_features(struct netfront_info *np);
135 static int xn_configure_features(struct netfront_info *np);
136 static void netif_free(struct netfront_info *info);
137 static int netfront_detach(device_t dev);
138 
139 static int xn_txq_mq_start_locked(struct netfront_txq *, struct mbuf *);
140 static int xn_txq_mq_start(if_t, struct mbuf *);
141 
142 static int talk_to_backend(device_t dev, struct netfront_info *info);
143 static int create_netdev(device_t dev);
144 static void netif_disconnect_backend(struct netfront_info *info);
145 static int setup_device(device_t dev, struct netfront_info *info,
146     unsigned long);
147 static int xn_ifmedia_upd(if_t ifp);
148 static void xn_ifmedia_sts(if_t ifp, struct ifmediareq *ifmr);
149 
150 static int xn_connect(struct netfront_info *);
151 static void xn_kick_rings(struct netfront_info *);
152 
153 static int xn_get_responses(struct netfront_rxq *,
154     struct netfront_rx_info *, RING_IDX, RING_IDX *,
155     struct mbuf **);
156 
157 #define virt_to_mfn(x) (vtophys(x) >> PAGE_SHIFT)
158 
159 #define INVALID_P2M_ENTRY (~0UL)
160 #define XN_QUEUE_NAME_LEN  8	/* xn{t,r}x_%u, allow for two digits */
161 struct netfront_rxq {
162 	struct netfront_info 	*info;
163 	u_int			id;
164 	char			name[XN_QUEUE_NAME_LEN];
165 	struct mtx		lock;
166 
167 	int			ring_ref;
168 	netif_rx_front_ring_t 	ring;
169 	xen_intr_handle_t	xen_intr_handle;
170 
171 	grant_ref_t 		gref_head;
172 	grant_ref_t 		grant_ref[NET_RX_RING_SIZE + 1];
173 
174 	struct mbuf		*mbufs[NET_RX_RING_SIZE + 1];
175 
176 	struct lro_ctrl		lro;
177 
178 	struct callout		rx_refill;
179 };
180 
181 struct netfront_txq {
182 	struct netfront_info 	*info;
183 	u_int 			id;
184 	char			name[XN_QUEUE_NAME_LEN];
185 	struct mtx		lock;
186 
187 	int			ring_ref;
188 	netif_tx_front_ring_t	ring;
189 	xen_intr_handle_t 	xen_intr_handle;
190 
191 	grant_ref_t		gref_head;
192 	grant_ref_t		grant_ref[NET_TX_RING_SIZE + 1];
193 
194 	struct mbuf		*mbufs[NET_TX_RING_SIZE + 1];
195 	int			mbufs_cnt;
196 	struct buf_ring		*br;
197 
198 	struct taskqueue 	*tq;
199 	struct task       	defrtask;
200 
201 	bus_dma_segment_t	segs[MAX_TX_REQ_FRAGS];
202 	struct mbuf_xennet {
203 		struct m_tag 	tag;
204 		bus_dma_tag_t	dma_tag;
205 		bus_dmamap_t	dma_map;
206 		struct netfront_txq *txq;
207 		SLIST_ENTRY(mbuf_xennet) next;
208 		u_int 		count;
209 	}			xennet_tag[NET_TX_RING_SIZE + 1];
210 	SLIST_HEAD(, mbuf_xennet) tags;
211 
212 	bool			full;
213 };
214 
215 struct netfront_info {
216 	if_t			xn_ifp;
217 
218 	struct mtx   		sc_lock;
219 
220 	u_int  num_queues;
221 	struct netfront_rxq 	*rxq;
222 	struct netfront_txq 	*txq;
223 
224 	u_int			carrier;
225 	u_int			maxfrags;
226 
227 	device_t		xbdev;
228 	uint8_t			mac[ETHER_ADDR_LEN];
229 
230 	int			xn_if_flags;
231 
232 	struct ifmedia		sc_media;
233 
234 	bus_dma_tag_t		dma_tag;
235 
236 	bool			xn_reset;
237 };
238 
239 struct netfront_rx_info {
240 	struct netif_rx_response rx;
241 	struct netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1];
242 };
243 
244 #define XN_RX_LOCK(_q)         mtx_lock(&(_q)->lock)
245 #define XN_RX_UNLOCK(_q)       mtx_unlock(&(_q)->lock)
246 
247 #define XN_TX_LOCK(_q)         mtx_lock(&(_q)->lock)
248 #define XN_TX_TRYLOCK(_q)      mtx_trylock(&(_q)->lock)
249 #define XN_TX_UNLOCK(_q)       mtx_unlock(&(_q)->lock)
250 
251 #define XN_LOCK(_sc)           mtx_lock(&(_sc)->sc_lock);
252 #define XN_UNLOCK(_sc)         mtx_unlock(&(_sc)->sc_lock);
253 
254 #define XN_LOCK_ASSERT(_sc)    mtx_assert(&(_sc)->sc_lock, MA_OWNED);
255 #define XN_RX_LOCK_ASSERT(_q)  mtx_assert(&(_q)->lock, MA_OWNED);
256 #define XN_TX_LOCK_ASSERT(_q)  mtx_assert(&(_q)->lock, MA_OWNED);
257 
258 #define netfront_carrier_on(netif)	((netif)->carrier = 1)
259 #define netfront_carrier_off(netif)	((netif)->carrier = 0)
260 #define netfront_carrier_ok(netif)	((netif)->carrier)
261 
262 /* Access macros for acquiring freeing slots in xn_free_{tx,rx}_idxs[]. */
263 
264 static inline void
265 add_id_to_freelist(struct mbuf **list, uintptr_t id)
266 {
267 
268 	KASSERT(id != 0,
269 		("%s: the head item (0) must always be free.", __func__));
270 	list[id] = list[0];
271 	list[0]  = (struct mbuf *)id;
272 }
273 
274 static inline unsigned short
275 get_id_from_freelist(struct mbuf **list)
276 {
277 	uintptr_t id;
278 
279 	id = (uintptr_t)list[0];
280 	KASSERT(id != 0,
281 		("%s: the head item (0) must always remain free.", __func__));
282 	list[0] = list[id];
283 	return (id);
284 }
285 
286 static inline int
287 xn_rxidx(RING_IDX idx)
288 {
289 
290 	return idx & (NET_RX_RING_SIZE - 1);
291 }
292 
293 static inline struct mbuf *
294 xn_get_rx_mbuf(struct netfront_rxq *rxq, RING_IDX ri)
295 {
296 	int i;
297 	struct mbuf *m;
298 
299 	i = xn_rxidx(ri);
300 	m = rxq->mbufs[i];
301 	rxq->mbufs[i] = NULL;
302 	return (m);
303 }
304 
305 static inline grant_ref_t
306 xn_get_rx_ref(struct netfront_rxq *rxq, RING_IDX ri)
307 {
308 	int i = xn_rxidx(ri);
309 	grant_ref_t ref = rxq->grant_ref[i];
310 
311 	KASSERT(ref != GRANT_REF_INVALID, ("Invalid grant reference!\n"));
312 	rxq->grant_ref[i] = GRANT_REF_INVALID;
313 	return (ref);
314 }
315 
316 #define MTAG_COOKIE 1218492000
317 #define MTAG_XENNET 0
318 
319 static void mbuf_grab(struct mbuf *m)
320 {
321 	struct mbuf_xennet *ref;
322 
323 	ref = (struct mbuf_xennet *)m_tag_locate(m, MTAG_COOKIE,
324 	    MTAG_XENNET, NULL);
325 	KASSERT(ref != NULL, ("Cannot find refcount"));
326 	ref->count++;
327 }
328 
329 static void mbuf_release(struct mbuf *m)
330 {
331 	struct mbuf_xennet *ref;
332 
333 	ref = (struct mbuf_xennet *)m_tag_locate(m, MTAG_COOKIE,
334 	    MTAG_XENNET, NULL);
335 	KASSERT(ref != NULL, ("Cannot find refcount"));
336 	KASSERT(ref->count > 0, ("Invalid reference count"));
337 
338 	if (--ref->count == 0) {
339 		/*
340 		 * Explicitly free the tag while we hold the tx queue lock.
341 		 * This ensures that the tag is deleted promptly in case
342 		 * something else is holding extra references to the mbuf chain,
343 		 * such as netmap.
344 		 */
345 		m_tag_delete(m, &ref->tag);
346 		m_freem(m);
347 	}
348 }
349 
350 static void tag_free(struct m_tag *t)
351 {
352 	struct mbuf_xennet *ref = (struct mbuf_xennet *)t;
353 
354 	KASSERT(ref->count == 0, ("Free mbuf tag with pending refcnt"));
355 	bus_dmamap_sync(ref->dma_tag, ref->dma_map, BUS_DMASYNC_POSTWRITE);
356 	bus_dmamap_destroy(ref->dma_tag, ref->dma_map);
357 	SLIST_INSERT_HEAD(&ref->txq->tags, ref, next);
358 }
359 
360 #define IPRINTK(fmt, args...) \
361     printf("[XEN] " fmt, ##args)
362 #ifdef INVARIANTS
363 #define WPRINTK(fmt, args...) \
364     printf("[XEN] " fmt, ##args)
365 #else
366 #define WPRINTK(fmt, args...)
367 #endif
368 #ifdef DEBUG
369 #define DPRINTK(fmt, args...) \
370     printf("[XEN] %s: " fmt, __func__, ##args)
371 #else
372 #define DPRINTK(fmt, args...)
373 #endif
374 
375 /**
376  * Read the 'mac' node at the given device's node in the store, and parse that
377  * as colon-separated octets, placing result the given mac array.  mac must be
378  * a preallocated array of length ETH_ALEN (as declared in linux/if_ether.h).
379  * Return 0 on success, or errno on error.
380  */
381 static int
382 xen_net_read_mac(device_t dev, uint8_t mac[])
383 {
384 	int error, i;
385 	char *s, *e, *macstr;
386 	const char *path;
387 
388 	path = xenbus_get_node(dev);
389 	error = xs_read(XST_NIL, path, "mac", NULL, (void **) &macstr);
390 	if (error == ENOENT) {
391 		/*
392 		 * Deal with missing mac XenStore nodes on devices with
393 		 * HVM emulation (the 'ioemu' configuration attribute)
394 		 * enabled.
395 		 *
396 		 * The HVM emulator may execute in a stub device model
397 		 * domain which lacks the permission, only given to Dom0,
398 		 * to update the guest's XenStore tree.  For this reason,
399 		 * the HVM emulator doesn't even attempt to write the
400 		 * front-side mac node, even when operating in Dom0.
401 		 * However, there should always be a mac listed in the
402 		 * backend tree.  Fallback to this version if our query
403 		 * of the front side XenStore location doesn't find
404 		 * anything.
405 		 */
406 		path = xenbus_get_otherend_path(dev);
407 		error = xs_read(XST_NIL, path, "mac", NULL, (void **) &macstr);
408 	}
409 	if (error != 0) {
410 		xenbus_dev_fatal(dev, error, "parsing %s/mac", path);
411 		return (error);
412 	}
413 
414 	s = macstr;
415 	for (i = 0; i < ETHER_ADDR_LEN; i++) {
416 		mac[i] = strtoul(s, &e, 16);
417 		if (s == e || (e[0] != ':' && e[0] != 0)) {
418 			free(macstr, M_XENBUS);
419 			return (ENOENT);
420 		}
421 		s = &e[1];
422 	}
423 	free(macstr, M_XENBUS);
424 	return (0);
425 }
426 
427 /**
428  * Entry point to this code when a new device is created.  Allocate the basic
429  * structures and the ring buffers for communication with the backend, and
430  * inform the backend of the appropriate details for those.  Switch to
431  * Connected state.
432  */
433 static int
434 netfront_probe(device_t dev)
435 {
436 
437 	if (xen_pv_nics_disabled())
438 		return (ENXIO);
439 
440 	if (!strcmp(xenbus_get_type(dev), "vif")) {
441 		device_set_desc(dev, "Virtual Network Interface");
442 		return (0);
443 	}
444 
445 	return (ENXIO);
446 }
447 
448 static int
449 netfront_attach(device_t dev)
450 {
451 	int err;
452 
453 	err = create_netdev(dev);
454 	if (err != 0) {
455 		xenbus_dev_fatal(dev, err, "creating netdev");
456 		return (err);
457 	}
458 
459 	SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
460 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
461 	    OID_AUTO, "enable_lro", CTLFLAG_RW,
462 	    &xn_enable_lro, 0, "Large Receive Offload");
463 
464 	SYSCTL_ADD_ULONG(device_get_sysctl_ctx(dev),
465 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
466 	    OID_AUTO, "num_queues", CTLFLAG_RD,
467 	    &xn_num_queues, "Number of pairs of queues");
468 
469 	return (0);
470 }
471 
472 static int
473 netfront_suspend(device_t dev)
474 {
475 	struct netfront_info *np = device_get_softc(dev);
476 	u_int i;
477 
478 	for (i = 0; i < np->num_queues; i++) {
479 		XN_RX_LOCK(&np->rxq[i]);
480 		XN_TX_LOCK(&np->txq[i]);
481 	}
482 	netfront_carrier_off(np);
483 	for (i = 0; i < np->num_queues; i++) {
484 		XN_RX_UNLOCK(&np->rxq[i]);
485 		XN_TX_UNLOCK(&np->txq[i]);
486 	}
487 	return (0);
488 }
489 
490 /**
491  * We are reconnecting to the backend, due to a suspend/resume, or a backend
492  * driver restart.  We tear down our netif structure and recreate it, but
493  * leave the device-layer structures intact so that this is transparent to the
494  * rest of the kernel.
495  */
496 static int
497 netfront_resume(device_t dev)
498 {
499 	struct netfront_info *info = device_get_softc(dev);
500 	u_int i;
501 
502 	if (xen_suspend_cancelled) {
503 		for (i = 0; i < info->num_queues; i++) {
504 			XN_RX_LOCK(&info->rxq[i]);
505 			XN_TX_LOCK(&info->txq[i]);
506 		}
507 		netfront_carrier_on(info);
508 		for (i = 0; i < info->num_queues; i++) {
509 			XN_RX_UNLOCK(&info->rxq[i]);
510 			XN_TX_UNLOCK(&info->txq[i]);
511 		}
512 		return (0);
513 	}
514 
515 	netif_disconnect_backend(info);
516 	return (0);
517 }
518 
519 static int
520 write_queue_xenstore_keys(device_t dev,
521     struct netfront_rxq *rxq,
522     struct netfront_txq *txq,
523     struct xs_transaction *xst, bool hierarchy)
524 {
525 	int err;
526 	const char *message;
527 	const char *node = xenbus_get_node(dev);
528 	char *path;
529 	size_t path_size;
530 
531 	KASSERT(rxq->id == txq->id, ("Mismatch between RX and TX queue ids"));
532 	/* Split event channel support is not yet there. */
533 	KASSERT(rxq->xen_intr_handle == txq->xen_intr_handle,
534 	    ("Split event channels are not supported"));
535 
536 	if (hierarchy) {
537 		path_size = strlen(node) + 10;
538 		path = malloc(path_size, M_DEVBUF, M_WAITOK|M_ZERO);
539 		snprintf(path, path_size, "%s/queue-%u", node, rxq->id);
540 	} else {
541 		path_size = strlen(node) + 1;
542 		path = malloc(path_size, M_DEVBUF, M_WAITOK|M_ZERO);
543 		snprintf(path, path_size, "%s", node);
544 	}
545 
546 	err = xs_printf(*xst, path, "tx-ring-ref","%u", txq->ring_ref);
547 	if (err != 0) {
548 		message = "writing tx ring-ref";
549 		goto error;
550 	}
551 	err = xs_printf(*xst, path, "rx-ring-ref","%u", rxq->ring_ref);
552 	if (err != 0) {
553 		message = "writing rx ring-ref";
554 		goto error;
555 	}
556 	err = xs_printf(*xst, path, "event-channel", "%u",
557 	    xen_intr_port(rxq->xen_intr_handle));
558 	if (err != 0) {
559 		message = "writing event-channel";
560 		goto error;
561 	}
562 
563 	free(path, M_DEVBUF);
564 
565 	return (0);
566 
567 error:
568 	free(path, M_DEVBUF);
569 	xenbus_dev_fatal(dev, err, "%s", message);
570 
571 	return (err);
572 }
573 
574 /* Common code used when first setting up, and when resuming. */
575 static int
576 talk_to_backend(device_t dev, struct netfront_info *info)
577 {
578 	const char *message;
579 	struct xs_transaction xst;
580 	const char *node = xenbus_get_node(dev);
581 	int err;
582 	unsigned long num_queues, max_queues = 0;
583 	unsigned int i;
584 
585 	err = xen_net_read_mac(dev, info->mac);
586 	if (err != 0) {
587 		xenbus_dev_fatal(dev, err, "parsing %s/mac", node);
588 		goto out;
589 	}
590 
591 	err = xs_scanf(XST_NIL, xenbus_get_otherend_path(info->xbdev),
592 	    "multi-queue-max-queues", NULL, "%lu", &max_queues);
593 	if (err != 0)
594 		max_queues = 1;
595 	num_queues = xn_num_queues;
596 	if (num_queues > max_queues)
597 		num_queues = max_queues;
598 
599 	err = setup_device(dev, info, num_queues);
600 	if (err != 0) {
601 		xenbus_dev_fatal(dev, err, "setup device");
602 		goto out;
603 	}
604 
605  again:
606 	err = xs_transaction_start(&xst);
607 	if (err != 0) {
608 		xenbus_dev_fatal(dev, err, "starting transaction");
609 		goto free;
610 	}
611 
612 	if (info->num_queues == 1) {
613 		err = write_queue_xenstore_keys(dev, &info->rxq[0],
614 		    &info->txq[0], &xst, false);
615 		if (err != 0)
616 			goto abort_transaction_no_def_error;
617 	} else {
618 		err = xs_printf(xst, node, "multi-queue-num-queues",
619 		    "%u", info->num_queues);
620 		if (err != 0) {
621 			message = "writing multi-queue-num-queues";
622 			goto abort_transaction;
623 		}
624 
625 		for (i = 0; i < info->num_queues; i++) {
626 			err = write_queue_xenstore_keys(dev, &info->rxq[i],
627 			    &info->txq[i], &xst, true);
628 			if (err != 0)
629 				goto abort_transaction_no_def_error;
630 		}
631 	}
632 
633 	err = xs_printf(xst, node, "request-rx-copy", "%u", 1);
634 	if (err != 0) {
635 		message = "writing request-rx-copy";
636 		goto abort_transaction;
637 	}
638 	err = xs_printf(xst, node, "feature-rx-notify", "%d", 1);
639 	if (err != 0) {
640 		message = "writing feature-rx-notify";
641 		goto abort_transaction;
642 	}
643 	err = xs_printf(xst, node, "feature-sg", "%d", 1);
644 	if (err != 0) {
645 		message = "writing feature-sg";
646 		goto abort_transaction;
647 	}
648 	if ((if_getcapenable(info->xn_ifp) & IFCAP_LRO) != 0) {
649 		err = xs_printf(xst, node, "feature-gso-tcpv4", "%d", 1);
650 		if (err != 0) {
651 			message = "writing feature-gso-tcpv4";
652 			goto abort_transaction;
653 		}
654 	}
655 	if ((if_getcapenable(info->xn_ifp) & IFCAP_RXCSUM) == 0) {
656 		err = xs_printf(xst, node, "feature-no-csum-offload", "%d", 1);
657 		if (err != 0) {
658 			message = "writing feature-no-csum-offload";
659 			goto abort_transaction;
660 		}
661 	}
662 
663 	err = xs_transaction_end(xst, 0);
664 	if (err != 0) {
665 		if (err == EAGAIN)
666 			goto again;
667 		xenbus_dev_fatal(dev, err, "completing transaction");
668 		goto free;
669 	}
670 
671 	return 0;
672 
673  abort_transaction:
674 	xenbus_dev_fatal(dev, err, "%s", message);
675  abort_transaction_no_def_error:
676 	xs_transaction_end(xst, 1);
677  free:
678 	netif_free(info);
679  out:
680 	return (err);
681 }
682 
683 static void
684 xn_rxq_intr(struct netfront_rxq *rxq)
685 {
686 
687 	XN_RX_LOCK(rxq);
688 	xn_rxeof(rxq);
689 	XN_RX_UNLOCK(rxq);
690 }
691 
692 static void
693 xn_txq_start(struct netfront_txq *txq)
694 {
695 	struct netfront_info *np = txq->info;
696 	if_t ifp = np->xn_ifp;
697 
698 	XN_TX_LOCK_ASSERT(txq);
699 	if (!drbr_empty(ifp, txq->br))
700 		xn_txq_mq_start_locked(txq, NULL);
701 }
702 
703 static void
704 xn_txq_intr(struct netfront_txq *txq)
705 {
706 
707 	XN_TX_LOCK(txq);
708 	if (RING_HAS_UNCONSUMED_RESPONSES(&txq->ring))
709 		xn_txeof(txq);
710 	xn_txq_start(txq);
711 	XN_TX_UNLOCK(txq);
712 }
713 
714 static void
715 xn_txq_tq_deferred(void *xtxq, int pending)
716 {
717 	struct netfront_txq *txq = xtxq;
718 
719 	XN_TX_LOCK(txq);
720 	xn_txq_start(txq);
721 	XN_TX_UNLOCK(txq);
722 }
723 
724 static void
725 disconnect_rxq(struct netfront_rxq *rxq)
726 {
727 
728 	xn_release_rx_bufs(rxq);
729 	gnttab_free_grant_references(rxq->gref_head);
730 	if (rxq->ring_ref != GRANT_REF_INVALID) {
731 		gnttab_end_foreign_access(rxq->ring_ref, NULL);
732 		rxq->ring_ref = GRANT_REF_INVALID;
733 	}
734 	/*
735 	 * No split event channel support at the moment, handle will
736 	 * be unbound in tx. So no need to call xen_intr_unbind here,
737 	 * but we do want to reset the handler to 0.
738 	 */
739 	rxq->xen_intr_handle = 0;
740 }
741 
742 static void
743 destroy_rxq(struct netfront_rxq *rxq)
744 {
745 
746 	callout_drain(&rxq->rx_refill);
747 	free(rxq->ring.sring, M_DEVBUF);
748 	rxq->ring.sring = NULL;
749 }
750 
751 static void
752 destroy_rxqs(struct netfront_info *np)
753 {
754 	int i;
755 
756 	for (i = 0; i < np->num_queues; i++)
757 		destroy_rxq(&np->rxq[i]);
758 
759 	free(np->rxq, M_DEVBUF);
760 	np->rxq = NULL;
761 }
762 
763 static int
764 setup_rxqs(device_t dev, struct netfront_info *info,
765 	   unsigned long num_queues)
766 {
767 	int q, i;
768 	int error;
769 	netif_rx_sring_t *rxs;
770 	struct netfront_rxq *rxq;
771 
772 	info->rxq = malloc(sizeof(struct netfront_rxq) * num_queues,
773 	    M_DEVBUF, M_WAITOK|M_ZERO);
774 
775 	for (q = 0; q < num_queues; q++) {
776 		rxq = &info->rxq[q];
777 
778 		rxq->id = q;
779 		rxq->info = info;
780 
781 		rxq->gref_head = GNTTAB_LIST_END;
782 		rxq->ring_ref = GRANT_REF_INVALID;
783 		rxq->ring.sring = NULL;
784 		snprintf(rxq->name, XN_QUEUE_NAME_LEN, "xnrx_%u", q);
785 		mtx_init(&rxq->lock, rxq->name, "netfront receive lock",
786 		    MTX_DEF);
787 
788 		for (i = 0; i <= NET_RX_RING_SIZE; i++) {
789 			rxq->mbufs[i] = NULL;
790 			rxq->grant_ref[i] = GRANT_REF_INVALID;
791 		}
792 
793 		/* Start resources allocation */
794 
795 		if (gnttab_alloc_grant_references(NET_RX_RING_SIZE,
796 		    &rxq->gref_head) != 0) {
797 			device_printf(dev, "allocating rx gref");
798 			error = ENOMEM;
799 			goto fail;
800 		}
801 
802 		rxs = (netif_rx_sring_t *)malloc(PAGE_SIZE, M_DEVBUF,
803 		    M_WAITOK|M_ZERO);
804 		SHARED_RING_INIT(rxs);
805 		FRONT_RING_INIT(&rxq->ring, rxs, PAGE_SIZE);
806 
807 		error = xenbus_grant_ring(dev, virt_to_mfn(rxs),
808 		    &rxq->ring_ref);
809 		if (error != 0) {
810 			device_printf(dev, "granting rx ring page");
811 			goto fail_grant_ring;
812 		}
813 
814 		callout_init(&rxq->rx_refill, 1);
815 	}
816 
817 	return (0);
818 
819 fail_grant_ring:
820 	gnttab_free_grant_references(rxq->gref_head);
821 	free(rxq->ring.sring, M_DEVBUF);
822 fail:
823 	for (; q >= 0; q--) {
824 		disconnect_rxq(&info->rxq[q]);
825 		destroy_rxq(&info->rxq[q]);
826 	}
827 
828 	free(info->rxq, M_DEVBUF);
829 	return (error);
830 }
831 
832 static void
833 disconnect_txq(struct netfront_txq *txq)
834 {
835 
836 	xn_release_tx_bufs(txq);
837 	gnttab_free_grant_references(txq->gref_head);
838 	if (txq->ring_ref != GRANT_REF_INVALID) {
839 		gnttab_end_foreign_access(txq->ring_ref, NULL);
840 		txq->ring_ref = GRANT_REF_INVALID;
841 	}
842 	xen_intr_unbind(&txq->xen_intr_handle);
843 }
844 
845 static void
846 destroy_txq(struct netfront_txq *txq)
847 {
848 	unsigned int i;
849 
850 	free(txq->ring.sring, M_DEVBUF);
851 	txq->ring.sring = NULL;
852 	buf_ring_free(txq->br, M_DEVBUF);
853 	txq->br = NULL;
854 	if (txq->tq) {
855 		taskqueue_drain_all(txq->tq);
856 		taskqueue_free(txq->tq);
857 		txq->tq = NULL;
858 	}
859 
860 	for (i = 0; i <= NET_TX_RING_SIZE; i++) {
861 		bus_dmamap_destroy(txq->info->dma_tag,
862 		    txq->xennet_tag[i].dma_map);
863 		txq->xennet_tag[i].dma_map = NULL;
864 	}
865 }
866 
867 static void
868 destroy_txqs(struct netfront_info *np)
869 {
870 	int i;
871 
872 	for (i = 0; i < np->num_queues; i++)
873 		destroy_txq(&np->txq[i]);
874 
875 	free(np->txq, M_DEVBUF);
876 	np->txq = NULL;
877 }
878 
879 static int
880 setup_txqs(device_t dev, struct netfront_info *info,
881 	   unsigned long num_queues)
882 {
883 	int q, i;
884 	int error;
885 	netif_tx_sring_t *txs;
886 	struct netfront_txq *txq;
887 
888 	info->txq = malloc(sizeof(struct netfront_txq) * num_queues,
889 	    M_DEVBUF, M_WAITOK|M_ZERO);
890 
891 	for (q = 0; q < num_queues; q++) {
892 		txq = &info->txq[q];
893 
894 		txq->id = q;
895 		txq->info = info;
896 
897 		txq->gref_head = GNTTAB_LIST_END;
898 		txq->ring_ref = GRANT_REF_INVALID;
899 		txq->ring.sring = NULL;
900 
901 		snprintf(txq->name, XN_QUEUE_NAME_LEN, "xntx_%u", q);
902 
903 		mtx_init(&txq->lock, txq->name, "netfront transmit lock",
904 		    MTX_DEF);
905 		SLIST_INIT(&txq->tags);
906 
907 		for (i = 0; i <= NET_TX_RING_SIZE; i++) {
908 			txq->mbufs[i] = (void *) ((u_long) i+1);
909 			txq->grant_ref[i] = GRANT_REF_INVALID;
910 			txq->xennet_tag[i].txq = txq;
911 			txq->xennet_tag[i].dma_tag = info->dma_tag;
912 			error = bus_dmamap_create(info->dma_tag, 0,
913 			    &txq->xennet_tag[i].dma_map);
914 			if (error != 0) {
915 				device_printf(dev,
916 				    "failed to allocate dma map\n");
917 				goto fail;
918 			}
919 			m_tag_setup(&txq->xennet_tag[i].tag,
920 			    MTAG_COOKIE, MTAG_XENNET,
921 			    sizeof(txq->xennet_tag[i]) -
922 			    sizeof(txq->xennet_tag[i].tag));
923 			txq->xennet_tag[i].tag.m_tag_free = &tag_free;
924 			SLIST_INSERT_HEAD(&txq->tags, &txq->xennet_tag[i],
925 			    next);
926 		}
927 		txq->mbufs[NET_TX_RING_SIZE] = (void *)0;
928 
929 		/* Start resources allocation. */
930 
931 		if (gnttab_alloc_grant_references(NET_TX_RING_SIZE,
932 		    &txq->gref_head) != 0) {
933 			device_printf(dev, "failed to allocate tx grant refs\n");
934 			error = ENOMEM;
935 			goto fail;
936 		}
937 
938 		txs = (netif_tx_sring_t *)malloc(PAGE_SIZE, M_DEVBUF,
939 		    M_WAITOK|M_ZERO);
940 		SHARED_RING_INIT(txs);
941 		FRONT_RING_INIT(&txq->ring, txs, PAGE_SIZE);
942 
943 		error = xenbus_grant_ring(dev, virt_to_mfn(txs),
944 		    &txq->ring_ref);
945 		if (error != 0) {
946 			device_printf(dev, "failed to grant tx ring\n");
947 			goto fail_grant_ring;
948 		}
949 
950 		txq->br = buf_ring_alloc(NET_TX_RING_SIZE, M_DEVBUF,
951 		    M_WAITOK, &txq->lock);
952 		TASK_INIT(&txq->defrtask, 0, xn_txq_tq_deferred, txq);
953 
954 		txq->tq = taskqueue_create(txq->name, M_WAITOK,
955 		    taskqueue_thread_enqueue, &txq->tq);
956 
957 		error = taskqueue_start_threads(&txq->tq, 1, PI_NET,
958 		    "%s txq %d", device_get_nameunit(dev), txq->id);
959 		if (error != 0) {
960 			device_printf(dev, "failed to start tx taskq %d\n",
961 			    txq->id);
962 			goto fail_start_thread;
963 		}
964 
965 		error = xen_intr_alloc_and_bind_local_port(dev,
966 		    xenbus_get_otherend_id(dev), /* filter */ NULL, xn_intr,
967 		    &info->txq[q], INTR_TYPE_NET | INTR_MPSAFE | INTR_ENTROPY,
968 		    &txq->xen_intr_handle);
969 
970 		if (error != 0) {
971 			device_printf(dev, "xen_intr_alloc_and_bind_local_port failed\n");
972 			goto fail_bind_port;
973 		}
974 	}
975 
976 	return (0);
977 
978 fail_bind_port:
979 	taskqueue_drain_all(txq->tq);
980 fail_start_thread:
981 	buf_ring_free(txq->br, M_DEVBUF);
982 	taskqueue_free(txq->tq);
983 	gnttab_end_foreign_access(txq->ring_ref, NULL);
984 fail_grant_ring:
985 	gnttab_free_grant_references(txq->gref_head);
986 	free(txq->ring.sring, M_DEVBUF);
987 fail:
988 	for (; q >= 0; q--) {
989 		disconnect_txq(&info->txq[q]);
990 		destroy_txq(&info->txq[q]);
991 	}
992 
993 	free(info->txq, M_DEVBUF);
994 	return (error);
995 }
996 
997 static int
998 setup_device(device_t dev, struct netfront_info *info,
999     unsigned long num_queues)
1000 {
1001 	int error;
1002 	int q;
1003 
1004 	if (info->txq)
1005 		destroy_txqs(info);
1006 
1007 	if (info->rxq)
1008 		destroy_rxqs(info);
1009 
1010 	info->num_queues = 0;
1011 
1012 	error = setup_rxqs(dev, info, num_queues);
1013 	if (error != 0)
1014 		goto out;
1015 	error = setup_txqs(dev, info, num_queues);
1016 	if (error != 0)
1017 		goto out;
1018 
1019 	info->num_queues = num_queues;
1020 
1021 	/* No split event channel at the moment. */
1022 	for (q = 0; q < num_queues; q++)
1023 		info->rxq[q].xen_intr_handle = info->txq[q].xen_intr_handle;
1024 
1025 	return (0);
1026 
1027 out:
1028 	KASSERT(error != 0, ("Error path taken without providing an error code"));
1029 	return (error);
1030 }
1031 
1032 #ifdef INET
1033 static u_int
1034 netfront_addr_cb(void *arg, struct ifaddr *a, u_int count)
1035 {
1036 	arp_ifinit((if_t)arg, a);
1037 	return (1);
1038 }
1039 /**
1040  * If this interface has an ipv4 address, send an arp for it. This
1041  * helps to get the network going again after migrating hosts.
1042  */
1043 static void
1044 netfront_send_fake_arp(device_t dev, struct netfront_info *info)
1045 {
1046 	if_t ifp;
1047 
1048 	ifp = info->xn_ifp;
1049 	if_foreach_addr_type(ifp, AF_INET, netfront_addr_cb, ifp);
1050 }
1051 #endif
1052 
1053 /**
1054  * Callback received when the backend's state changes.
1055  */
1056 static void
1057 netfront_backend_changed(device_t dev, XenbusState newstate)
1058 {
1059 	struct netfront_info *sc = device_get_softc(dev);
1060 
1061 	DPRINTK("newstate=%d\n", newstate);
1062 
1063 	CURVNET_SET(if_getvnet(sc->xn_ifp));
1064 
1065 	switch (newstate) {
1066 	case XenbusStateInitialising:
1067 	case XenbusStateInitialised:
1068 	case XenbusStateUnknown:
1069 	case XenbusStateReconfigured:
1070 	case XenbusStateReconfiguring:
1071 		break;
1072 	case XenbusStateInitWait:
1073 		if (xenbus_get_state(dev) != XenbusStateInitialising)
1074 			break;
1075 		if (xn_connect(sc) != 0)
1076 			break;
1077 		/* Switch to connected state before kicking the rings. */
1078 		xenbus_set_state(sc->xbdev, XenbusStateConnected);
1079 		xn_kick_rings(sc);
1080 		break;
1081 	case XenbusStateClosing:
1082 		xenbus_set_state(dev, XenbusStateClosed);
1083 		break;
1084 	case XenbusStateClosed:
1085 		if (sc->xn_reset) {
1086 			netif_disconnect_backend(sc);
1087 			xenbus_set_state(dev, XenbusStateInitialising);
1088 			sc->xn_reset = false;
1089 		}
1090 		break;
1091 	case XenbusStateConnected:
1092 #ifdef INET
1093 		netfront_send_fake_arp(dev, sc);
1094 #endif
1095 		break;
1096 	}
1097 
1098 	CURVNET_RESTORE();
1099 }
1100 
1101 /**
1102  * \brief Verify that there is sufficient space in the Tx ring
1103  *        buffer for a maximally sized request to be enqueued.
1104  *
1105  * A transmit request requires a transmit descriptor for each packet
1106  * fragment, plus up to 2 entries for "options" (e.g. TSO).
1107  */
1108 static inline int
1109 xn_tx_slot_available(struct netfront_txq *txq)
1110 {
1111 
1112 	return (RING_FREE_REQUESTS(&txq->ring) > (MAX_TX_REQ_FRAGS + 2));
1113 }
1114 
1115 static void
1116 xn_release_tx_bufs(struct netfront_txq *txq)
1117 {
1118 	int i;
1119 
1120 	for (i = 1; i <= NET_TX_RING_SIZE; i++) {
1121 		struct mbuf *m;
1122 
1123 		m = txq->mbufs[i];
1124 
1125 		/*
1126 		 * We assume that no kernel addresses are
1127 		 * less than NET_TX_RING_SIZE.  Any entry
1128 		 * in the table that is below this number
1129 		 * must be an index from free-list tracking.
1130 		 */
1131 		if (((uintptr_t)m) <= NET_TX_RING_SIZE)
1132 			continue;
1133 		gnttab_end_foreign_access_ref(txq->grant_ref[i]);
1134 		gnttab_release_grant_reference(&txq->gref_head,
1135 		    txq->grant_ref[i]);
1136 		txq->grant_ref[i] = GRANT_REF_INVALID;
1137 		add_id_to_freelist(txq->mbufs, i);
1138 		txq->mbufs_cnt--;
1139 		if (txq->mbufs_cnt < 0) {
1140 			panic("%s: tx_chain_cnt must be >= 0", __func__);
1141 		}
1142 		mbuf_release(m);
1143 	}
1144 }
1145 
1146 static struct mbuf *
1147 xn_alloc_one_rx_buffer(struct netfront_rxq *rxq)
1148 {
1149 	struct mbuf *m;
1150 
1151 	m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MJUMPAGESIZE);
1152 	if (m == NULL)
1153 		return NULL;
1154 	m->m_len = m->m_pkthdr.len = MJUMPAGESIZE;
1155 
1156 	return (m);
1157 }
1158 
1159 static void
1160 xn_alloc_rx_buffers(struct netfront_rxq *rxq)
1161 {
1162 	RING_IDX req_prod;
1163 	int notify;
1164 
1165 	XN_RX_LOCK_ASSERT(rxq);
1166 
1167 	if (__predict_false(rxq->info->carrier == 0))
1168 		return;
1169 
1170 	for (req_prod = rxq->ring.req_prod_pvt;
1171 	     req_prod - rxq->ring.rsp_cons < NET_RX_RING_SIZE;
1172 	     req_prod++) {
1173 		struct mbuf *m;
1174 		unsigned short id;
1175 		grant_ref_t ref;
1176 		struct netif_rx_request *req;
1177 		unsigned long pfn;
1178 
1179 		m = xn_alloc_one_rx_buffer(rxq);
1180 		if (m == NULL)
1181 			break;
1182 
1183 		id = xn_rxidx(req_prod);
1184 
1185 		KASSERT(rxq->mbufs[id] == NULL, ("non-NULL xn_rx_chain"));
1186 		rxq->mbufs[id] = m;
1187 
1188 		ref = gnttab_claim_grant_reference(&rxq->gref_head);
1189 		KASSERT(ref != GNTTAB_LIST_END,
1190 		    ("reserved grant references exhuasted"));
1191 		rxq->grant_ref[id] = ref;
1192 
1193 		pfn = atop(vtophys(mtod(m, vm_offset_t)));
1194 		req = RING_GET_REQUEST(&rxq->ring, req_prod);
1195 
1196 		gnttab_grant_foreign_access_ref(ref,
1197 		    xenbus_get_otherend_id(rxq->info->xbdev), pfn, 0);
1198 		req->id = id;
1199 		req->gref = ref;
1200 	}
1201 
1202 	rxq->ring.req_prod_pvt = req_prod;
1203 
1204 	/* Not enough requests? Try again later. */
1205 	if (req_prod - rxq->ring.rsp_cons < NET_RX_SLOTS_MIN) {
1206 		callout_reset_curcpu(&rxq->rx_refill, hz/10,
1207 		    xn_alloc_rx_buffers_callout, rxq);
1208 		return;
1209 	}
1210 
1211 	wmb();		/* barrier so backend seens requests */
1212 
1213 	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&rxq->ring, notify);
1214 	if (notify)
1215 		xen_intr_signal(rxq->xen_intr_handle);
1216 }
1217 
1218 static void xn_alloc_rx_buffers_callout(void *arg)
1219 {
1220 	struct netfront_rxq *rxq;
1221 
1222 	rxq = (struct netfront_rxq *)arg;
1223 	XN_RX_LOCK(rxq);
1224 	xn_alloc_rx_buffers(rxq);
1225 	XN_RX_UNLOCK(rxq);
1226 }
1227 
1228 static void
1229 xn_release_rx_bufs(struct netfront_rxq *rxq)
1230 {
1231 	int i,  ref;
1232 	struct mbuf *m;
1233 
1234 	for (i = 0; i < NET_RX_RING_SIZE; i++) {
1235 		m = rxq->mbufs[i];
1236 
1237 		if (m == NULL)
1238 			continue;
1239 
1240 		ref = rxq->grant_ref[i];
1241 		if (ref == GRANT_REF_INVALID)
1242 			continue;
1243 
1244 		gnttab_end_foreign_access_ref(ref);
1245 		gnttab_release_grant_reference(&rxq->gref_head, ref);
1246 		rxq->mbufs[i] = NULL;
1247 		rxq->grant_ref[i] = GRANT_REF_INVALID;
1248 		m_freem(m);
1249 	}
1250 }
1251 
1252 static void
1253 xn_rxeof(struct netfront_rxq *rxq)
1254 {
1255 	if_t ifp;
1256 	struct netfront_info *np = rxq->info;
1257 #if (defined(INET) || defined(INET6))
1258 	struct lro_ctrl *lro = &rxq->lro;
1259 #endif
1260 	struct netfront_rx_info rinfo;
1261 	struct netif_rx_response *rx = &rinfo.rx;
1262 	struct netif_extra_info *extras = rinfo.extras;
1263 	RING_IDX i, rp;
1264 	struct mbuf *m;
1265 	struct mbufq mbufq_rxq, mbufq_errq;
1266 	int err, work_to_do;
1267 
1268 	XN_RX_LOCK_ASSERT(rxq);
1269 
1270 	if (!netfront_carrier_ok(np))
1271 		return;
1272 
1273 	/* XXX: there should be some sane limit. */
1274 	mbufq_init(&mbufq_errq, INT_MAX);
1275 	mbufq_init(&mbufq_rxq, INT_MAX);
1276 
1277 	ifp = np->xn_ifp;
1278 
1279 	do {
1280 		rp = rxq->ring.sring->rsp_prod;
1281 		rmb();	/* Ensure we see queued responses up to 'rp'. */
1282 
1283 		i = rxq->ring.rsp_cons;
1284 		while ((i != rp)) {
1285 			memcpy(rx, RING_GET_RESPONSE(&rxq->ring, i), sizeof(*rx));
1286 			memset(extras, 0, sizeof(rinfo.extras));
1287 
1288 			m = NULL;
1289 			err = xn_get_responses(rxq, &rinfo, rp, &i, &m);
1290 
1291 			if (__predict_false(err)) {
1292 				if (m)
1293 					(void )mbufq_enqueue(&mbufq_errq, m);
1294 				if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1);
1295 				continue;
1296 			}
1297 
1298 			m->m_pkthdr.rcvif = ifp;
1299 			if (rx->flags & NETRXF_data_validated) {
1300 				/*
1301 				 * According to mbuf(9) the correct way to tell
1302 				 * the stack that the checksum of an inbound
1303 				 * packet is correct, without it actually being
1304 				 * present (because the underlying interface
1305 				 * doesn't provide it), is to set the
1306 				 * CSUM_DATA_VALID and CSUM_PSEUDO_HDR flags,
1307 				 * and the csum_data field to 0xffff.
1308 				 */
1309 				m->m_pkthdr.csum_flags |= (CSUM_DATA_VALID
1310 				    | CSUM_PSEUDO_HDR);
1311 				m->m_pkthdr.csum_data = 0xffff;
1312 			}
1313 			if ((rx->flags & NETRXF_extra_info) != 0 &&
1314 			    (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type ==
1315 			    XEN_NETIF_EXTRA_TYPE_GSO)) {
1316 				m->m_pkthdr.tso_segsz =
1317 				extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].u.gso.size;
1318 				m->m_pkthdr.csum_flags |= CSUM_TSO;
1319 			}
1320 
1321 			(void )mbufq_enqueue(&mbufq_rxq, m);
1322 		}
1323 
1324 		rxq->ring.rsp_cons = i;
1325 
1326 		xn_alloc_rx_buffers(rxq);
1327 
1328 		RING_FINAL_CHECK_FOR_RESPONSES(&rxq->ring, work_to_do);
1329 	} while (work_to_do);
1330 
1331 	mbufq_drain(&mbufq_errq);
1332 	/*
1333 	 * Process all the mbufs after the remapping is complete.
1334 	 * Break the mbuf chain first though.
1335 	 */
1336 	while ((m = mbufq_dequeue(&mbufq_rxq)) != NULL) {
1337 		if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
1338 #if (defined(INET) || defined(INET6))
1339 		/* Use LRO if possible */
1340 		if ((if_getcapenable(ifp) & IFCAP_LRO) == 0 ||
1341 		    lro->lro_cnt == 0 || tcp_lro_rx(lro, m, 0)) {
1342 			/*
1343 			 * If LRO fails, pass up to the stack
1344 			 * directly.
1345 			 */
1346 			if_input(ifp, m);
1347 		}
1348 #else
1349 		if_input(ifp, m);
1350 #endif
1351 	}
1352 
1353 #if (defined(INET) || defined(INET6))
1354 	/*
1355 	 * Flush any outstanding LRO work
1356 	 */
1357 	tcp_lro_flush_all(lro);
1358 #endif
1359 }
1360 
1361 static void
1362 xn_txeof(struct netfront_txq *txq)
1363 {
1364 	RING_IDX i, prod;
1365 	unsigned short id;
1366 	if_t ifp;
1367 	netif_tx_response_t *txr;
1368 	struct mbuf *m;
1369 	struct netfront_info *np = txq->info;
1370 
1371 	XN_TX_LOCK_ASSERT(txq);
1372 
1373 	if (!netfront_carrier_ok(np))
1374 		return;
1375 
1376 	ifp = np->xn_ifp;
1377 
1378 	do {
1379 		prod = txq->ring.sring->rsp_prod;
1380 		rmb(); /* Ensure we see responses up to 'rp'. */
1381 
1382 		for (i = txq->ring.rsp_cons; i != prod; i++) {
1383 			txr = RING_GET_RESPONSE(&txq->ring, i);
1384 			if (txr->status == NETIF_RSP_NULL)
1385 				continue;
1386 
1387 			if (txr->status != NETIF_RSP_OKAY) {
1388 				printf("%s: WARNING: response is %d!\n",
1389 				       __func__, txr->status);
1390 			}
1391 			id = txr->id;
1392 			m = txq->mbufs[id];
1393 			KASSERT(m != NULL, ("mbuf not found in chain"));
1394 			KASSERT((uintptr_t)m > NET_TX_RING_SIZE,
1395 				("mbuf already on the free list, but we're "
1396 				"trying to free it again!"));
1397 			M_ASSERTVALID(m);
1398 
1399 			if (__predict_false(gnttab_query_foreign_access(
1400 			    txq->grant_ref[id]) != 0)) {
1401 				panic("%s: grant id %u still in use by the "
1402 				    "backend", __func__, id);
1403 			}
1404 			gnttab_end_foreign_access_ref(txq->grant_ref[id]);
1405 			gnttab_release_grant_reference(
1406 				&txq->gref_head, txq->grant_ref[id]);
1407 			txq->grant_ref[id] = GRANT_REF_INVALID;
1408 
1409 			txq->mbufs[id] = NULL;
1410 			add_id_to_freelist(txq->mbufs, id);
1411 			txq->mbufs_cnt--;
1412 			mbuf_release(m);
1413 			/* Only mark the txq active if we've freed up at least one slot to try */
1414 			if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE);
1415 		}
1416 		txq->ring.rsp_cons = prod;
1417 
1418 		/*
1419 		 * Set a new event, then check for race with update of
1420 		 * tx_cons. Note that it is essential to schedule a
1421 		 * callback, no matter how few buffers are pending. Even if
1422 		 * there is space in the transmit ring, higher layers may
1423 		 * be blocked because too much data is outstanding: in such
1424 		 * cases notification from Xen is likely to be the only kick
1425 		 * that we'll get.
1426 		 */
1427 		txq->ring.sring->rsp_event =
1428 		    prod + ((txq->ring.sring->req_prod - prod) >> 1) + 1;
1429 
1430 		mb();
1431 	} while (prod != txq->ring.sring->rsp_prod);
1432 
1433 	if (txq->full &&
1434 	    ((txq->ring.sring->req_prod - prod) < NET_TX_RING_SIZE)) {
1435 		txq->full = false;
1436 		xn_txq_start(txq);
1437 	}
1438 }
1439 
1440 static void
1441 xn_intr(void *xsc)
1442 {
1443 	struct netfront_txq *txq = xsc;
1444 	struct netfront_info *np = txq->info;
1445 	struct netfront_rxq *rxq = &np->rxq[txq->id];
1446 
1447 	/* kick both tx and rx */
1448 	xn_rxq_intr(rxq);
1449 	xn_txq_intr(txq);
1450 }
1451 
1452 static void
1453 xn_move_rx_slot(struct netfront_rxq *rxq, struct mbuf *m,
1454     grant_ref_t ref)
1455 {
1456 	int new = xn_rxidx(rxq->ring.req_prod_pvt);
1457 
1458 	KASSERT(rxq->mbufs[new] == NULL, ("mbufs != NULL"));
1459 	rxq->mbufs[new] = m;
1460 	rxq->grant_ref[new] = ref;
1461 	RING_GET_REQUEST(&rxq->ring, rxq->ring.req_prod_pvt)->id = new;
1462 	RING_GET_REQUEST(&rxq->ring, rxq->ring.req_prod_pvt)->gref = ref;
1463 	rxq->ring.req_prod_pvt++;
1464 }
1465 
1466 static int
1467 xn_get_extras(struct netfront_rxq *rxq,
1468     struct netif_extra_info *extras, RING_IDX rp, RING_IDX *cons)
1469 {
1470 	struct netif_extra_info *extra;
1471 
1472 	int err = 0;
1473 
1474 	do {
1475 		struct mbuf *m;
1476 		grant_ref_t ref;
1477 
1478 		if (__predict_false(*cons + 1 == rp)) {
1479 			err = EINVAL;
1480 			break;
1481 		}
1482 
1483 		extra = (struct netif_extra_info *)
1484 		RING_GET_RESPONSE(&rxq->ring, ++(*cons));
1485 
1486 		if (__predict_false(!extra->type ||
1487 			extra->type >= XEN_NETIF_EXTRA_TYPE_MAX)) {
1488 			err = EINVAL;
1489 		} else {
1490 			memcpy(&extras[extra->type - 1], extra, sizeof(*extra));
1491 		}
1492 
1493 		m = xn_get_rx_mbuf(rxq, *cons);
1494 		ref = xn_get_rx_ref(rxq,  *cons);
1495 		xn_move_rx_slot(rxq, m, ref);
1496 	} while (extra->flags & XEN_NETIF_EXTRA_FLAG_MORE);
1497 
1498 	return err;
1499 }
1500 
1501 static int
1502 xn_get_responses(struct netfront_rxq *rxq,
1503     struct netfront_rx_info *rinfo, RING_IDX rp, RING_IDX *cons,
1504     struct mbuf  **list)
1505 {
1506 	struct netif_rx_response *rx = &rinfo->rx;
1507 	struct netif_extra_info *extras = rinfo->extras;
1508 	struct mbuf *m, *m0, *m_prev;
1509 	grant_ref_t ref = xn_get_rx_ref(rxq, *cons);
1510 	int frags = 1;
1511 	int err = 0;
1512 	u_long ret __diagused;
1513 
1514 	m0 = m = m_prev = xn_get_rx_mbuf(rxq, *cons);
1515 
1516 	if (rx->flags & NETRXF_extra_info) {
1517 		err = xn_get_extras(rxq, extras, rp, cons);
1518 	}
1519 
1520 	if (m0 != NULL) {
1521 		m0->m_pkthdr.len = 0;
1522 		m0->m_next = NULL;
1523 	}
1524 
1525 	for (;;) {
1526 #if 0
1527 		DPRINTK("rx->status=%hd rx->offset=%hu frags=%u\n",
1528 			rx->status, rx->offset, frags);
1529 #endif
1530 		if (__predict_false(rx->status < 0 ||
1531 			rx->offset + rx->status > PAGE_SIZE)) {
1532 			xn_move_rx_slot(rxq, m, ref);
1533 			if (m0 == m)
1534 				m0 = NULL;
1535 			m = NULL;
1536 			err = EINVAL;
1537 			goto next_skip_queue;
1538 		}
1539 
1540 		/*
1541 		 * This definitely indicates a bug, either in this driver or in
1542 		 * the backend driver. In future this should flag the bad
1543 		 * situation to the system controller to reboot the backed.
1544 		 */
1545 		if (ref == GRANT_REF_INVALID) {
1546 			printf("%s: Bad rx response id %d.\n", __func__, rx->id);
1547 			err = EINVAL;
1548 			goto next;
1549 		}
1550 
1551 		ret = gnttab_end_foreign_access_ref(ref);
1552 		KASSERT(ret, ("Unable to end access to grant references"));
1553 
1554 		gnttab_release_grant_reference(&rxq->gref_head, ref);
1555 
1556 next:
1557 		if (m == NULL)
1558 			break;
1559 
1560 		m->m_len = rx->status;
1561 		m->m_data += rx->offset;
1562 		m0->m_pkthdr.len += rx->status;
1563 
1564 next_skip_queue:
1565 		if (!(rx->flags & NETRXF_more_data))
1566 			break;
1567 
1568 		if (*cons + frags == rp) {
1569 			if (net_ratelimit())
1570 				WPRINTK("Need more frags\n");
1571 			err = ENOENT;
1572 			printf("%s: cons %u frags %u rp %u, not enough frags\n",
1573 			       __func__, *cons, frags, rp);
1574 			break;
1575 		}
1576 		/*
1577 		 * Note that m can be NULL, if rx->status < 0 or if
1578 		 * rx->offset + rx->status > PAGE_SIZE above.
1579 		 */
1580 		m_prev = m;
1581 
1582 		rx = RING_GET_RESPONSE(&rxq->ring, *cons + frags);
1583 		m = xn_get_rx_mbuf(rxq, *cons + frags);
1584 
1585 		/*
1586 		 * m_prev == NULL can happen if rx->status < 0 or if
1587 		 * rx->offset + * rx->status > PAGE_SIZE above.
1588 		 */
1589 		if (m_prev != NULL)
1590 			m_prev->m_next = m;
1591 
1592 		/*
1593 		 * m0 can be NULL if rx->status < 0 or if * rx->offset +
1594 		 * rx->status > PAGE_SIZE above.
1595 		 */
1596 		if (m0 == NULL)
1597 			m0 = m;
1598 		m->m_next = NULL;
1599 		ref = xn_get_rx_ref(rxq, *cons + frags);
1600 		frags++;
1601 	}
1602 	*list = m0;
1603 	*cons += frags;
1604 
1605 	return (err);
1606 }
1607 
1608 /**
1609  * Given an mbuf chain, make sure we have enough room and then push
1610  * it onto the transmit ring.
1611  */
1612 static int
1613 xn_assemble_tx_request(struct netfront_txq *txq, struct mbuf *m_head)
1614 {
1615 	struct netfront_info *np = txq->info;
1616 	if_t ifp = np->xn_ifp;
1617 	int otherend_id, error, nfrags;
1618 	bus_dma_segment_t *segs = txq->segs;
1619 	struct mbuf_xennet *tag;
1620 	bus_dmamap_t map;
1621 	unsigned int i;
1622 
1623 	KASSERT(!SLIST_EMPTY(&txq->tags), ("no tags available"));
1624 	tag = SLIST_FIRST(&txq->tags);
1625 	SLIST_REMOVE_HEAD(&txq->tags, next);
1626 	KASSERT(tag->count == 0, ("tag already in-use"));
1627 	map = tag->dma_map;
1628 	error = bus_dmamap_load_mbuf_sg(np->dma_tag, map, m_head, segs,
1629 	    &nfrags, 0);
1630 	if (error == EFBIG || nfrags > np->maxfrags) {
1631 		struct mbuf *m;
1632 
1633 		bus_dmamap_unload(np->dma_tag, map);
1634 		m = m_defrag(m_head, M_NOWAIT);
1635 		if (!m) {
1636 			/*
1637 			 * Defrag failed, so free the mbuf and
1638 			 * therefore drop the packet.
1639 			 */
1640 			SLIST_INSERT_HEAD(&txq->tags, tag, next);
1641 			m_freem(m_head);
1642 			return (EMSGSIZE);
1643 		}
1644 		m_head = m;
1645 		error = bus_dmamap_load_mbuf_sg(np->dma_tag, map, m_head, segs,
1646 		    &nfrags, 0);
1647 		if (error != 0 || nfrags > np->maxfrags) {
1648 			bus_dmamap_unload(np->dma_tag, map);
1649 			SLIST_INSERT_HEAD(&txq->tags, tag, next);
1650 			m_freem(m_head);
1651 			return (error ?: EFBIG);
1652 		}
1653 	} else if (error != 0) {
1654 		SLIST_INSERT_HEAD(&txq->tags, tag, next);
1655 		m_freem(m_head);
1656 		return (error);
1657 	}
1658 
1659 	/**
1660 	 * The FreeBSD TCP stack, with TSO enabled, can produce a chain
1661 	 * of mbufs longer than Linux can handle.  Make sure we don't
1662 	 * pass a too-long chain over to the other side by dropping the
1663 	 * packet.  It doesn't look like there is currently a way to
1664 	 * tell the TCP stack to generate a shorter chain of packets.
1665 	 */
1666 	if (nfrags > MAX_TX_REQ_FRAGS) {
1667 #ifdef DEBUG
1668 		printf("%s: nfrags %d > MAX_TX_REQ_FRAGS %d, netback "
1669 		       "won't be able to handle it, dropping\n",
1670 		       __func__, nfrags, MAX_TX_REQ_FRAGS);
1671 #endif
1672 		SLIST_INSERT_HEAD(&txq->tags, tag, next);
1673 		bus_dmamap_unload(np->dma_tag, map);
1674 		m_freem(m_head);
1675 		return (EMSGSIZE);
1676 	}
1677 
1678 	/*
1679 	 * This check should be redundant.  We've already verified that we
1680 	 * have enough slots in the ring to handle a packet of maximum
1681 	 * size, and that our packet is less than the maximum size.  Keep
1682 	 * it in here as an assert for now just to make certain that
1683 	 * chain_cnt is accurate.
1684 	 */
1685 	KASSERT((txq->mbufs_cnt + nfrags) <= NET_TX_RING_SIZE,
1686 		("%s: chain_cnt (%d) + nfrags (%d) > NET_TX_RING_SIZE "
1687 		 "(%d)!", __func__, (int) txq->mbufs_cnt,
1688                     (int) nfrags, (int) NET_TX_RING_SIZE));
1689 
1690 	/*
1691 	 * Start packing the mbufs in this chain into
1692 	 * the fragment pointers. Stop when we run out
1693 	 * of fragments or hit the end of the mbuf chain.
1694 	 */
1695 	otherend_id = xenbus_get_otherend_id(np->xbdev);
1696 	m_tag_prepend(m_head, &tag->tag);
1697 	for (i = 0; i < nfrags; i++) {
1698 		netif_tx_request_t *tx;
1699 		uintptr_t id;
1700 		grant_ref_t ref;
1701 		u_long mfn; /* XXX Wrong type? */
1702 
1703 		tx = RING_GET_REQUEST(&txq->ring, txq->ring.req_prod_pvt);
1704 		id = get_id_from_freelist(txq->mbufs);
1705 		if (id == 0)
1706 			panic("%s: was allocated the freelist head!\n",
1707 			    __func__);
1708 		txq->mbufs_cnt++;
1709 		if (txq->mbufs_cnt > NET_TX_RING_SIZE)
1710 			panic("%s: tx_chain_cnt must be <= NET_TX_RING_SIZE\n",
1711 			    __func__);
1712 		mbuf_grab(m_head);
1713 		txq->mbufs[id] = m_head;
1714 		tx->id = id;
1715 		ref = gnttab_claim_grant_reference(&txq->gref_head);
1716 		KASSERT((short)ref >= 0, ("Negative ref"));
1717 		mfn = atop(segs[i].ds_addr);
1718 		gnttab_grant_foreign_access_ref(ref, otherend_id,
1719 		    mfn, GNTMAP_readonly);
1720 		tx->gref = txq->grant_ref[id] = ref;
1721 		tx->offset = segs[i].ds_addr & PAGE_MASK;
1722 		KASSERT(tx->offset + segs[i].ds_len <= PAGE_SIZE,
1723 		    ("mbuf segment crosses a page boundary"));
1724 		tx->flags = 0;
1725 		if (i == 0) {
1726 			/*
1727 			 * The first fragment has the entire packet
1728 			 * size, subsequent fragments have just the
1729 			 * fragment size. The backend works out the
1730 			 * true size of the first fragment by
1731 			 * subtracting the sizes of the other
1732 			 * fragments.
1733 			 */
1734 			tx->size = m_head->m_pkthdr.len;
1735 
1736 			/*
1737 			 * The first fragment contains the checksum flags
1738 			 * and is optionally followed by extra data for
1739 			 * TSO etc.
1740 			 */
1741 			/**
1742 			 * CSUM_TSO requires checksum offloading.
1743 			 * Some versions of FreeBSD fail to
1744 			 * set CSUM_TCP in the CSUM_TSO case,
1745 			 * so we have to test for CSUM_TSO
1746 			 * explicitly.
1747 			 */
1748 			if (m_head->m_pkthdr.csum_flags
1749 			    & (CSUM_DELAY_DATA | CSUM_TSO)) {
1750 				tx->flags |= (NETTXF_csum_blank
1751 				    | NETTXF_data_validated);
1752 			}
1753 			if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1754 				struct netif_extra_info *gso =
1755 					(struct netif_extra_info *)
1756 					RING_GET_REQUEST(&txq->ring,
1757 							 ++txq->ring.req_prod_pvt);
1758 
1759 				tx->flags |= NETTXF_extra_info;
1760 
1761 				gso->u.gso.size = m_head->m_pkthdr.tso_segsz;
1762 				gso->u.gso.type =
1763 					XEN_NETIF_GSO_TYPE_TCPV4;
1764 				gso->u.gso.pad = 0;
1765 				gso->u.gso.features = 0;
1766 
1767 				gso->type = XEN_NETIF_EXTRA_TYPE_GSO;
1768 				gso->flags = 0;
1769 			}
1770 		} else {
1771 			tx->size = segs[i].ds_len;
1772 		}
1773 		if (i != nfrags - 1)
1774 			tx->flags |= NETTXF_more_data;
1775 
1776 		txq->ring.req_prod_pvt++;
1777 	}
1778 	bus_dmamap_sync(np->dma_tag, map, BUS_DMASYNC_PREWRITE);
1779 	BPF_MTAP(ifp, m_head);
1780 
1781 	if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
1782 	if_inc_counter(ifp, IFCOUNTER_OBYTES, m_head->m_pkthdr.len);
1783 	if (m_head->m_flags & M_MCAST)
1784 		if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
1785 
1786 	xn_txeof(txq);
1787 
1788 	return (0);
1789 }
1790 
1791 /* equivalent of network_open() in Linux */
1792 static void
1793 xn_ifinit_locked(struct netfront_info *np)
1794 {
1795 	if_t ifp;
1796 	int i;
1797 	struct netfront_rxq *rxq;
1798 
1799 	XN_LOCK_ASSERT(np);
1800 
1801 	ifp = np->xn_ifp;
1802 
1803 	if (if_getdrvflags(ifp) & IFF_DRV_RUNNING || !netfront_carrier_ok(np))
1804 		return;
1805 
1806 	xn_stop(np);
1807 
1808 	for (i = 0; i < np->num_queues; i++) {
1809 		rxq = &np->rxq[i];
1810 		XN_RX_LOCK(rxq);
1811 		xn_alloc_rx_buffers(rxq);
1812 		rxq->ring.sring->rsp_event = rxq->ring.rsp_cons + 1;
1813 		if (RING_HAS_UNCONSUMED_RESPONSES(&rxq->ring))
1814 			xn_rxeof(rxq);
1815 		XN_RX_UNLOCK(rxq);
1816 	}
1817 
1818 	if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0);
1819 	if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE);
1820 	if_link_state_change(ifp, LINK_STATE_UP);
1821 }
1822 
1823 static void
1824 xn_ifinit(void *xsc)
1825 {
1826 	struct netfront_info *sc = xsc;
1827 
1828 	XN_LOCK(sc);
1829 	xn_ifinit_locked(sc);
1830 	XN_UNLOCK(sc);
1831 }
1832 
1833 static int
1834 xn_ioctl(if_t ifp, u_long cmd, caddr_t data)
1835 {
1836 	struct netfront_info *sc = if_getsoftc(ifp);
1837 	struct ifreq *ifr = (struct ifreq *) data;
1838 	device_t dev;
1839 #ifdef INET
1840 	struct ifaddr *ifa = (struct ifaddr *)data;
1841 #endif
1842 	int mask, error = 0, reinit;
1843 
1844 	dev = sc->xbdev;
1845 
1846 	switch(cmd) {
1847 	case SIOCSIFADDR:
1848 #ifdef INET
1849 		XN_LOCK(sc);
1850 		if (ifa->ifa_addr->sa_family == AF_INET) {
1851 			if_setflagbits(ifp, IFF_UP, 0);
1852 			if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))
1853 				xn_ifinit_locked(sc);
1854 			arp_ifinit(ifp, ifa);
1855 			XN_UNLOCK(sc);
1856 		} else {
1857 			XN_UNLOCK(sc);
1858 #endif
1859 			error = ether_ioctl(ifp, cmd, data);
1860 #ifdef INET
1861 		}
1862 #endif
1863 		break;
1864 	case SIOCSIFMTU:
1865 		if (if_getmtu(ifp) == ifr->ifr_mtu)
1866 			break;
1867 
1868 		if_setmtu(ifp, ifr->ifr_mtu);
1869 		if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
1870 		xn_ifinit(sc);
1871 		break;
1872 	case SIOCSIFFLAGS:
1873 		XN_LOCK(sc);
1874 		if (if_getflags(ifp) & IFF_UP) {
1875 			/*
1876 			 * If only the state of the PROMISC flag changed,
1877 			 * then just use the 'set promisc mode' command
1878 			 * instead of reinitializing the entire NIC. Doing
1879 			 * a full re-init means reloading the firmware and
1880 			 * waiting for it to start up, which may take a
1881 			 * second or two.
1882 			 */
1883 			xn_ifinit_locked(sc);
1884 		} else {
1885 			if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1886 				xn_stop(sc);
1887 			}
1888 		}
1889 		sc->xn_if_flags = if_getflags(ifp);
1890 		XN_UNLOCK(sc);
1891 		break;
1892 	case SIOCSIFCAP:
1893 		mask = ifr->ifr_reqcap ^ if_getcapenable(ifp);
1894 		reinit = 0;
1895 
1896 		if (mask & IFCAP_TXCSUM) {
1897 			if_togglecapenable(ifp, IFCAP_TXCSUM);
1898 			if_togglehwassist(ifp, XN_CSUM_FEATURES);
1899 		}
1900 		if (mask & IFCAP_TSO4) {
1901 			if_togglecapenable(ifp, IFCAP_TSO4);
1902 			if_togglehwassist(ifp, CSUM_TSO);
1903 		}
1904 
1905 		if (mask & (IFCAP_RXCSUM | IFCAP_LRO)) {
1906 			/* These Rx features require us to renegotiate. */
1907 			reinit = 1;
1908 
1909 			if (mask & IFCAP_RXCSUM)
1910 				if_togglecapenable(ifp, IFCAP_RXCSUM);
1911 			if (mask & IFCAP_LRO)
1912 				if_togglecapenable(ifp, IFCAP_LRO);
1913 		}
1914 
1915 		if (reinit == 0)
1916 			break;
1917 
1918 		/*
1919 		 * We must reset the interface so the backend picks up the
1920 		 * new features.
1921 		 */
1922 		device_printf(sc->xbdev,
1923 		    "performing interface reset due to feature change\n");
1924 		XN_LOCK(sc);
1925 		netfront_carrier_off(sc);
1926 		sc->xn_reset = true;
1927 		/*
1928 		 * NB: the pending packet queue is not flushed, since
1929 		 * the interface should still support the old options.
1930 		 */
1931 		XN_UNLOCK(sc);
1932 		/*
1933 		 * Delete the xenstore nodes that export features.
1934 		 *
1935 		 * NB: There's a xenbus state called
1936 		 * "XenbusStateReconfiguring", which is what we should set
1937 		 * here. Sadly none of the backends know how to handle it,
1938 		 * and simply disconnect from the frontend, so we will just
1939 		 * switch back to XenbusStateInitialising in order to force
1940 		 * a reconnection.
1941 		 */
1942 		xs_rm(XST_NIL, xenbus_get_node(dev), "feature-gso-tcpv4");
1943 		xs_rm(XST_NIL, xenbus_get_node(dev), "feature-no-csum-offload");
1944 		xenbus_set_state(dev, XenbusStateClosing);
1945 
1946 		/*
1947 		 * Wait for the frontend to reconnect before returning
1948 		 * from the ioctl. 30s should be more than enough for any
1949 		 * sane backend to reconnect.
1950 		 */
1951 		error = tsleep(sc, 0, "xn_rst", 30*hz);
1952 		break;
1953 	case SIOCADDMULTI:
1954 	case SIOCDELMULTI:
1955 		break;
1956 	case SIOCSIFMEDIA:
1957 	case SIOCGIFMEDIA:
1958 		error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd);
1959 		break;
1960 	default:
1961 		error = ether_ioctl(ifp, cmd, data);
1962 	}
1963 
1964 	return (error);
1965 }
1966 
1967 static void
1968 xn_stop(struct netfront_info *sc)
1969 {
1970 	if_t ifp;
1971 
1972 	XN_LOCK_ASSERT(sc);
1973 
1974 	ifp = sc->xn_ifp;
1975 
1976 	if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1977 	if_link_state_change(ifp, LINK_STATE_DOWN);
1978 }
1979 
1980 static void
1981 xn_rebuild_rx_bufs(struct netfront_rxq *rxq)
1982 {
1983 	int requeue_idx, i;
1984 	grant_ref_t ref;
1985 	netif_rx_request_t *req;
1986 
1987 	for (requeue_idx = 0, i = 0; i < NET_RX_RING_SIZE; i++) {
1988 		struct mbuf *m;
1989 		u_long pfn;
1990 
1991 		if (rxq->mbufs[i] == NULL)
1992 			continue;
1993 
1994 		m = rxq->mbufs[requeue_idx] = xn_get_rx_mbuf(rxq, i);
1995 		ref = rxq->grant_ref[requeue_idx] = xn_get_rx_ref(rxq, i);
1996 
1997 		req = RING_GET_REQUEST(&rxq->ring, requeue_idx);
1998 		pfn = vtophys(mtod(m, vm_offset_t)) >> PAGE_SHIFT;
1999 
2000 		gnttab_grant_foreign_access_ref(ref,
2001 		    xenbus_get_otherend_id(rxq->info->xbdev),
2002 		    pfn, 0);
2003 
2004 		req->gref = ref;
2005 		req->id   = requeue_idx;
2006 
2007 		requeue_idx++;
2008 	}
2009 
2010 	rxq->ring.req_prod_pvt = requeue_idx;
2011 }
2012 
2013 /* START of Xenolinux helper functions adapted to FreeBSD */
2014 static int
2015 xn_connect(struct netfront_info *np)
2016 {
2017 	int i, error;
2018 	u_int feature_rx_copy;
2019 	struct netfront_rxq *rxq;
2020 	struct netfront_txq *txq;
2021 
2022 	error = xs_scanf(XST_NIL, xenbus_get_otherend_path(np->xbdev),
2023 	    "feature-rx-copy", NULL, "%u", &feature_rx_copy);
2024 	if (error != 0)
2025 		feature_rx_copy = 0;
2026 
2027 	/* We only support rx copy. */
2028 	if (!feature_rx_copy)
2029 		return (EPROTONOSUPPORT);
2030 
2031 	/* Recovery procedure: */
2032 	error = talk_to_backend(np->xbdev, np);
2033 	if (error != 0)
2034 		return (error);
2035 
2036 	/* Step 1: Reinitialise variables. */
2037 	xn_query_features(np);
2038 	xn_configure_features(np);
2039 
2040 	/* Step 2: Release TX buffer */
2041 	for (i = 0; i < np->num_queues; i++) {
2042 		txq = &np->txq[i];
2043 		xn_release_tx_bufs(txq);
2044 	}
2045 
2046 	/* Step 3: Rebuild the RX buffer freelist and the RX ring itself. */
2047 	for (i = 0; i < np->num_queues; i++) {
2048 		rxq = &np->rxq[i];
2049 		xn_rebuild_rx_bufs(rxq);
2050 	}
2051 
2052 	/* Step 4: All public and private state should now be sane.  Get
2053 	 * ready to start sending and receiving packets and give the driver
2054 	 * domain a kick because we've probably just requeued some
2055 	 * packets.
2056 	 */
2057 	netfront_carrier_on(np);
2058 	wakeup(np);
2059 
2060 	return (0);
2061 }
2062 
2063 static void
2064 xn_kick_rings(struct netfront_info *np)
2065 {
2066 	struct netfront_rxq *rxq;
2067 	struct netfront_txq *txq;
2068 	int i;
2069 
2070 	for (i = 0; i < np->num_queues; i++) {
2071 		txq = &np->txq[i];
2072 		rxq = &np->rxq[i];
2073 		xen_intr_signal(txq->xen_intr_handle);
2074 		XN_TX_LOCK(txq);
2075 		xn_txeof(txq);
2076 		XN_TX_UNLOCK(txq);
2077 		XN_RX_LOCK(rxq);
2078 		xn_alloc_rx_buffers(rxq);
2079 		XN_RX_UNLOCK(rxq);
2080 	}
2081 }
2082 
2083 static void
2084 xn_query_features(struct netfront_info *np)
2085 {
2086 	int val;
2087 
2088 	device_printf(np->xbdev, "backend features:");
2089 
2090 	if (xs_scanf(XST_NIL, xenbus_get_otherend_path(np->xbdev),
2091 		"feature-sg", NULL, "%d", &val) != 0)
2092 		val = 0;
2093 
2094 	np->maxfrags = 1;
2095 	if (val) {
2096 		np->maxfrags = MAX_TX_REQ_FRAGS;
2097 		printf(" feature-sg");
2098 	}
2099 
2100 	if (xs_scanf(XST_NIL, xenbus_get_otherend_path(np->xbdev),
2101 		"feature-gso-tcpv4", NULL, "%d", &val) != 0)
2102 		val = 0;
2103 
2104 	if_setcapabilitiesbit(np->xn_ifp, 0, IFCAP_TSO4 | IFCAP_LRO);
2105 	if (val) {
2106 		if_setcapabilitiesbit(np->xn_ifp, IFCAP_TSO4 | IFCAP_LRO, 0);
2107 		printf(" feature-gso-tcp4");
2108 	}
2109 
2110 	/*
2111 	 * HW CSUM offload is assumed to be available unless
2112 	 * feature-no-csum-offload is set in xenstore.
2113 	 */
2114 	if (xs_scanf(XST_NIL, xenbus_get_otherend_path(np->xbdev),
2115 		"feature-no-csum-offload", NULL, "%d", &val) != 0)
2116 		val = 0;
2117 
2118 	if_setcapabilitiesbit(np->xn_ifp, IFCAP_HWCSUM, 0);
2119 	if (val) {
2120 		if_setcapabilitiesbit(np->xn_ifp, 0, IFCAP_HWCSUM);
2121 		printf(" feature-no-csum-offload");
2122 	}
2123 
2124 	printf("\n");
2125 }
2126 
2127 static int
2128 xn_configure_features(struct netfront_info *np)
2129 {
2130 	int err, cap_enabled;
2131 #if (defined(INET) || defined(INET6))
2132 	int i;
2133 #endif
2134 	if_t ifp;
2135 
2136 	ifp = np->xn_ifp;
2137 	err = 0;
2138 
2139 	if ((if_getcapenable(ifp) & if_getcapabilities(ifp)) == if_getcapenable(ifp)) {
2140 		/* Current options are available, no need to do anything. */
2141 		return (0);
2142 	}
2143 
2144 	/* Try to preserve as many options as possible. */
2145 	cap_enabled = if_getcapenable(ifp);
2146 	if_setcapenable(ifp, 0);
2147 	if_sethwassist(ifp, 0);
2148 
2149 #if (defined(INET) || defined(INET6))
2150 	if ((cap_enabled & IFCAP_LRO) != 0)
2151 		for (i = 0; i < np->num_queues; i++)
2152 			tcp_lro_free(&np->rxq[i].lro);
2153 	if (xn_enable_lro &&
2154 	    (if_getcapabilities(ifp) & cap_enabled & IFCAP_LRO) != 0) {
2155 	    	if_setcapenablebit(ifp, IFCAP_LRO, 0);
2156 		for (i = 0; i < np->num_queues; i++) {
2157 			err = tcp_lro_init(&np->rxq[i].lro);
2158 			if (err != 0) {
2159 				device_printf(np->xbdev,
2160 				    "LRO initialization failed\n");
2161 				if_setcapenablebit(ifp, 0, IFCAP_LRO);
2162 				break;
2163 			}
2164 			np->rxq[i].lro.ifp = ifp;
2165 		}
2166 	}
2167 	if ((if_getcapabilities(ifp) & cap_enabled & IFCAP_TSO4) != 0) {
2168 		if_setcapenablebit(ifp, IFCAP_TSO4, 0);
2169 		if_sethwassistbits(ifp, CSUM_TSO, 0);
2170 	}
2171 #endif
2172 	if ((if_getcapabilities(ifp) & cap_enabled & IFCAP_TXCSUM) != 0) {
2173 		if_setcapenablebit(ifp, IFCAP_TXCSUM, 0);
2174 		if_sethwassistbits(ifp, XN_CSUM_FEATURES, 0);
2175 	}
2176 	if ((if_getcapabilities(ifp) & cap_enabled & IFCAP_RXCSUM) != 0)
2177 		if_setcapenablebit(ifp, IFCAP_RXCSUM, 0);
2178 
2179 	return (err);
2180 }
2181 
2182 static int
2183 xn_txq_mq_start_locked(struct netfront_txq *txq, struct mbuf *m)
2184 {
2185 	struct netfront_info *np;
2186 	if_t ifp;
2187 	struct buf_ring *br;
2188 	int error, notify;
2189 
2190 	np = txq->info;
2191 	br = txq->br;
2192 	ifp = np->xn_ifp;
2193 	error = 0;
2194 
2195 	XN_TX_LOCK_ASSERT(txq);
2196 
2197 	if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0 ||
2198 	    !netfront_carrier_ok(np)) {
2199 		if (m != NULL)
2200 			error = drbr_enqueue(ifp, br, m);
2201 		return (error);
2202 	}
2203 
2204 	if (m != NULL) {
2205 		error = drbr_enqueue(ifp, br, m);
2206 		if (error != 0)
2207 			return (error);
2208 	}
2209 
2210 	while ((m = drbr_peek(ifp, br)) != NULL) {
2211 		if (!xn_tx_slot_available(txq)) {
2212 			drbr_putback(ifp, br, m);
2213 			break;
2214 		}
2215 
2216 		error = xn_assemble_tx_request(txq, m);
2217 		/* xn_assemble_tx_request always consumes the mbuf*/
2218 		if (error != 0) {
2219 			drbr_advance(ifp, br);
2220 			break;
2221 		}
2222 
2223 		RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&txq->ring, notify);
2224 		if (notify)
2225 			xen_intr_signal(txq->xen_intr_handle);
2226 
2227 		drbr_advance(ifp, br);
2228 	}
2229 
2230 	if (RING_FULL(&txq->ring))
2231 		txq->full = true;
2232 
2233 	return (0);
2234 }
2235 
2236 static int
2237 xn_txq_mq_start(if_t ifp, struct mbuf *m)
2238 {
2239 	struct netfront_info *np;
2240 	struct netfront_txq *txq;
2241 	int i, npairs, error;
2242 
2243 	np = if_getsoftc(ifp);
2244 	npairs = np->num_queues;
2245 
2246 	if (!netfront_carrier_ok(np))
2247 		return (ENOBUFS);
2248 
2249 	KASSERT(npairs != 0, ("called with 0 available queues"));
2250 
2251 	/* check if flowid is set */
2252 	if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
2253 		i = m->m_pkthdr.flowid % npairs;
2254 	else
2255 		i = curcpu % npairs;
2256 
2257 	txq = &np->txq[i];
2258 
2259 	if (XN_TX_TRYLOCK(txq) != 0) {
2260 		error = xn_txq_mq_start_locked(txq, m);
2261 		XN_TX_UNLOCK(txq);
2262 	} else {
2263 		error = drbr_enqueue(ifp, txq->br, m);
2264 		taskqueue_enqueue(txq->tq, &txq->defrtask);
2265 	}
2266 
2267 	return (error);
2268 }
2269 
2270 static void
2271 xn_qflush(if_t ifp)
2272 {
2273 	struct netfront_info *np;
2274 	struct netfront_txq *txq;
2275 	struct mbuf *m;
2276 	int i;
2277 
2278 	np = if_getsoftc(ifp);
2279 
2280 	for (i = 0; i < np->num_queues; i++) {
2281 		txq = &np->txq[i];
2282 
2283 		XN_TX_LOCK(txq);
2284 		while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
2285 			m_freem(m);
2286 		XN_TX_UNLOCK(txq);
2287 	}
2288 
2289 	if_qflush(ifp);
2290 }
2291 
2292 /**
2293  * Create a network device.
2294  * @param dev  Newbus device representing this virtual NIC.
2295  */
2296 int
2297 create_netdev(device_t dev)
2298 {
2299 	struct netfront_info *np;
2300 	int err, cap_enabled;
2301 	if_t ifp;
2302 
2303 	np = device_get_softc(dev);
2304 
2305 	np->xbdev         = dev;
2306 
2307 	mtx_init(&np->sc_lock, "xnsc", "netfront softc lock", MTX_DEF);
2308 
2309 	ifmedia_init(&np->sc_media, 0, xn_ifmedia_upd, xn_ifmedia_sts);
2310 	ifmedia_add(&np->sc_media, IFM_ETHER|IFM_MANUAL, 0, NULL);
2311 	ifmedia_set(&np->sc_media, IFM_ETHER|IFM_MANUAL);
2312 
2313 	err = xen_net_read_mac(dev, np->mac);
2314 	if (err != 0)
2315 		goto error;
2316 
2317 	/* Set up ifnet structure */
2318 	ifp = np->xn_ifp = if_alloc(IFT_ETHER);
2319 	if_setsoftc(ifp, np);
2320 	if_initname(ifp, "xn",  device_get_unit(dev));
2321 	if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
2322 	if_setioctlfn(ifp, xn_ioctl);
2323 
2324 	if_settransmitfn(ifp, xn_txq_mq_start);
2325 	if_setqflushfn(ifp, xn_qflush);
2326 
2327 	if_setinitfn(ifp, xn_ifinit);
2328 
2329 	if_sethwassist(ifp, XN_CSUM_FEATURES);
2330 	/* Enable all supported features at device creation. */
2331 	if_setcapabilities(ifp, IFCAP_HWCSUM|IFCAP_TSO4|IFCAP_LRO);
2332 	cap_enabled = if_getcapabilities(ifp);
2333 	if (!xn_enable_lro) {
2334 		cap_enabled &= ~IFCAP_LRO;
2335 	}
2336 	if_setcapenable(ifp, cap_enabled);
2337 
2338 	if_sethwtsomax(ifp, 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN));
2339 	if_sethwtsomaxsegcount(ifp, MAX_TX_REQ_FRAGS);
2340 	if_sethwtsomaxsegsize(ifp, PAGE_SIZE);
2341 
2342 	ether_ifattach(ifp, np->mac);
2343 	netfront_carrier_off(np);
2344 
2345 	err = bus_dma_tag_create(
2346 	    bus_get_dma_tag(dev),		/* parent */
2347 	    1, PAGE_SIZE,			/* algnmnt, boundary */
2348 	    BUS_SPACE_MAXADDR,			/* lowaddr */
2349 	    BUS_SPACE_MAXADDR,			/* highaddr */
2350 	    NULL, NULL,				/* filter, filterarg */
2351 	    PAGE_SIZE * MAX_TX_REQ_FRAGS,	/* max request size */
2352 	    MAX_TX_REQ_FRAGS,			/* max segments */
2353 	    PAGE_SIZE,				/* maxsegsize */
2354 	    BUS_DMA_ALLOCNOW,			/* flags */
2355 	    NULL, NULL,				/* lockfunc, lockarg */
2356 	    &np->dma_tag);
2357 
2358 	return (err);
2359 
2360 error:
2361 	KASSERT(err != 0, ("Error path with no error code specified"));
2362 	return (err);
2363 }
2364 
2365 static int
2366 netfront_detach(device_t dev)
2367 {
2368 	struct netfront_info *info = device_get_softc(dev);
2369 
2370 	DPRINTK("%s\n", xenbus_get_node(dev));
2371 
2372 	netif_free(info);
2373 
2374 	return 0;
2375 }
2376 
2377 static void
2378 netif_free(struct netfront_info *np)
2379 {
2380 
2381 	XN_LOCK(np);
2382 	xn_stop(np);
2383 	XN_UNLOCK(np);
2384 	netif_disconnect_backend(np);
2385 	ether_ifdetach(np->xn_ifp);
2386 	free(np->rxq, M_DEVBUF);
2387 	free(np->txq, M_DEVBUF);
2388 	if_free(np->xn_ifp);
2389 	np->xn_ifp = NULL;
2390 	ifmedia_removeall(&np->sc_media);
2391 	bus_dma_tag_destroy(np->dma_tag);
2392 }
2393 
2394 static void
2395 netif_disconnect_backend(struct netfront_info *np)
2396 {
2397 	u_int i;
2398 
2399 	for (i = 0; i < np->num_queues; i++) {
2400 		XN_RX_LOCK(&np->rxq[i]);
2401 		XN_TX_LOCK(&np->txq[i]);
2402 	}
2403 	netfront_carrier_off(np);
2404 	for (i = 0; i < np->num_queues; i++) {
2405 		XN_RX_UNLOCK(&np->rxq[i]);
2406 		XN_TX_UNLOCK(&np->txq[i]);
2407 	}
2408 
2409 	for (i = 0; i < np->num_queues; i++) {
2410 		disconnect_rxq(&np->rxq[i]);
2411 		disconnect_txq(&np->txq[i]);
2412 	}
2413 }
2414 
2415 static int
2416 xn_ifmedia_upd(if_t ifp)
2417 {
2418 
2419 	return (0);
2420 }
2421 
2422 static void
2423 xn_ifmedia_sts(if_t ifp, struct ifmediareq *ifmr)
2424 {
2425 
2426 	ifmr->ifm_status = IFM_AVALID|IFM_ACTIVE;
2427 	ifmr->ifm_active = IFM_ETHER|IFM_MANUAL;
2428 }
2429 
2430 /* ** Driver registration ** */
2431 static device_method_t netfront_methods[] = {
2432 	/* Device interface */
2433 	DEVMETHOD(device_probe,         netfront_probe),
2434 	DEVMETHOD(device_attach,        netfront_attach),
2435 	DEVMETHOD(device_detach,        netfront_detach),
2436 	DEVMETHOD(device_shutdown,      bus_generic_shutdown),
2437 	DEVMETHOD(device_suspend,       netfront_suspend),
2438 	DEVMETHOD(device_resume,        netfront_resume),
2439 
2440 	/* Xenbus interface */
2441 	DEVMETHOD(xenbus_otherend_changed, netfront_backend_changed),
2442 
2443 	DEVMETHOD_END
2444 };
2445 
2446 static driver_t netfront_driver = {
2447 	"xn",
2448 	netfront_methods,
2449 	sizeof(struct netfront_info),
2450 };
2451 
2452 DRIVER_MODULE(xe, xenbusb_front, netfront_driver, NULL, NULL);
2453