xref: /freebsd/sys/net/iflib.c (revision 55620f43deef5c0eb5b4b0f675de18b30c8d1c2d)
1 /*-
2  * Copyright (c) 2014-2016, Matthew Macy <mmacy@nextbsd.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are met:
7  *
8  *  1. Redistributions of source code must retain the above copyright notice,
9  *     this list of conditions and the following disclaimer.
10  *
11  *  2. Neither the name of Matthew Macy nor the names of its
12  *     contributors may be used to endorse or promote products derived from
13  *     this software without specific prior written permission.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
19  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
20  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
21  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
22  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
24  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
25  * POSSIBILITY OF SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30 
31 #include "opt_inet.h"
32 #include "opt_inet6.h"
33 #include "opt_acpi.h"
34 
35 #include <sys/param.h>
36 #include <sys/types.h>
37 #include <sys/bus.h>
38 #include <sys/eventhandler.h>
39 #include <sys/sockio.h>
40 #include <sys/kernel.h>
41 #include <sys/lock.h>
42 #include <sys/mutex.h>
43 #include <sys/module.h>
44 #include <sys/kobj.h>
45 #include <sys/rman.h>
46 #include <sys/sbuf.h>
47 #include <sys/smp.h>
48 #include <sys/socket.h>
49 #include <sys/sysctl.h>
50 #include <sys/syslog.h>
51 #include <sys/taskqueue.h>
52 
53 
54 #include <net/if.h>
55 #include <net/if_var.h>
56 #include <net/if_types.h>
57 #include <net/if_media.h>
58 #include <net/bpf.h>
59 #include <net/ethernet.h>
60 #include <net/mp_ring.h>
61 
62 #include <netinet/in.h>
63 #include <netinet/in_pcb.h>
64 #include <netinet/tcp_lro.h>
65 #include <netinet/in_systm.h>
66 #include <netinet/if_ether.h>
67 #include <netinet/ip.h>
68 #include <netinet/ip6.h>
69 #include <netinet/tcp.h>
70 
71 #include <machine/bus.h>
72 #include <machine/in_cksum.h>
73 
74 #include <vm/vm.h>
75 #include <vm/pmap.h>
76 
77 #include <dev/led/led.h>
78 #include <dev/pci/pcireg.h>
79 #include <dev/pci/pcivar.h>
80 #include <dev/pci/pci_private.h>
81 
82 #include <net/iflib.h>
83 
84 #include "ifdi_if.h"
85 
86 #if defined(__i386__) || defined(__amd64__)
87 #include <sys/memdesc.h>
88 #include <machine/bus.h>
89 #include <machine/md_var.h>
90 #include <machine/specialreg.h>
91 #include <x86/include/busdma_impl.h>
92 #include <x86/iommu/busdma_dmar.h>
93 #endif
94 
95 
96 /*
97  * enable accounting of every mbuf as it comes in to and goes out of iflib's software descriptor references
98  */
99 #define MEMORY_LOGGING 0
100 /*
101  * Enable mbuf vectors for compressing long mbuf chains
102  */
103 
104 
105 /*
106  * NB:
107  * - Prefetching in tx cleaning should perhaps be a tunable. The distance ahead
108  *   we prefetch needs to be determined by the time spent in m_free vis a vis
109  *   the cost of a prefetch. This will of course vary based on the workload:
110  *      - NFLX's m_free path is dominated by vm-based M_EXT manipulation which
111  *        is quite expensive, thus suggesting very little prefetch.
112  *      - small packet forwarding which is just returning a single mbuf to
113  *        UMA will typically be very fast vis a vis the cost of a memory
114  *        access.
115  */
116 
117 
118 /*
119  * File organization:
120  *  - private structures
121  *  - iflib private utility functions
122  *  - ifnet functions
123  *  - vlan registry and other exported functions
124  *  - iflib public core functions
125  *
126  *
127  */
128 static MALLOC_DEFINE(M_IFLIB, "iflib", "ifnet library");
129 
130 struct iflib_txq;
131 typedef struct iflib_txq *iflib_txq_t;
132 struct iflib_rxq;
133 typedef struct iflib_rxq *iflib_rxq_t;
134 struct iflib_fl;
135 typedef struct iflib_fl *iflib_fl_t;
136 
137 typedef struct iflib_filter_info {
138 	driver_filter_t *ifi_filter;
139 	void *ifi_filter_arg;
140 	struct grouptask *ifi_task;
141 } *iflib_filter_info_t;
142 
143 struct iflib_ctx {
144 	KOBJ_FIELDS;
145    /*
146    * Pointer to hardware driver's softc
147    */
148 	void *ifc_softc;
149 	device_t ifc_dev;
150 	if_t ifc_ifp;
151 
152 	cpuset_t ifc_cpus;
153 	if_shared_ctx_t ifc_sctx;
154 	struct if_softc_ctx ifc_softc_ctx;
155 
156 	struct mtx ifc_mtx;
157 
158 	uint16_t ifc_nhwtxqs;
159 	uint16_t ifc_nhwrxqs;
160 
161 	iflib_txq_t ifc_txqs;
162 	iflib_rxq_t ifc_rxqs;
163 	uint32_t ifc_if_flags;
164 	uint32_t ifc_flags;
165 	uint32_t ifc_max_fl_buf_size;
166 	int ifc_in_detach;
167 
168 	int ifc_link_state;
169 	int ifc_link_irq;
170 	int ifc_pause_frames;
171 	int ifc_watchdog_events;
172 	struct cdev *ifc_led_dev;
173 	struct resource *ifc_msix_mem;
174 
175 	struct if_irq ifc_legacy_irq;
176 	struct grouptask ifc_admin_task;
177 	struct grouptask ifc_vflr_task;
178 	struct iflib_filter_info ifc_filter_info;
179 	struct ifmedia	ifc_media;
180 
181 	struct sysctl_oid *ifc_sysctl_node;
182 	uint16_t ifc_sysctl_ntxqs;
183 	uint16_t ifc_sysctl_nrxqs;
184 	uint16_t ifc_sysctl_ntxds;
185 	uint16_t ifc_sysctl_nrxds;
186 	struct if_txrx ifc_txrx;
187 #define isc_txd_encap  ifc_txrx.ift_txd_encap
188 #define isc_txd_flush  ifc_txrx.ift_txd_flush
189 #define isc_txd_credits_update  ifc_txrx.ift_txd_credits_update
190 #define isc_rxd_available ifc_txrx.ift_rxd_available
191 #define isc_rxd_pkt_get ifc_txrx.ift_rxd_pkt_get
192 #define isc_rxd_refill ifc_txrx.ift_rxd_refill
193 #define isc_rxd_flush ifc_txrx.ift_rxd_flush
194 #define isc_rxd_refill ifc_txrx.ift_rxd_refill
195 #define isc_rxd_refill ifc_txrx.ift_rxd_refill
196 #define isc_legacy_intr ifc_txrx.ift_legacy_intr
197 	eventhandler_tag ifc_vlan_attach_event;
198 	eventhandler_tag ifc_vlan_detach_event;
199 	uint8_t ifc_mac[ETHER_ADDR_LEN];
200 	char ifc_mtx_name[16];
201 };
202 
203 
204 void *
205 iflib_get_softc(if_ctx_t ctx)
206 {
207 
208 	return (ctx->ifc_softc);
209 }
210 
211 device_t
212 iflib_get_dev(if_ctx_t ctx)
213 {
214 
215 	return (ctx->ifc_dev);
216 }
217 
218 if_t
219 iflib_get_ifp(if_ctx_t ctx)
220 {
221 
222 	return (ctx->ifc_ifp);
223 }
224 
225 struct ifmedia *
226 iflib_get_media(if_ctx_t ctx)
227 {
228 
229 	return (&ctx->ifc_media);
230 }
231 
232 void
233 iflib_set_mac(if_ctx_t ctx, uint8_t mac[ETHER_ADDR_LEN])
234 {
235 
236 	bcopy(mac, ctx->ifc_mac, ETHER_ADDR_LEN);
237 }
238 
239 if_softc_ctx_t
240 iflib_get_softc_ctx(if_ctx_t ctx)
241 {
242 
243 	return (&ctx->ifc_softc_ctx);
244 }
245 
246 if_shared_ctx_t
247 iflib_get_sctx(if_ctx_t ctx)
248 {
249 
250 	return (ctx->ifc_sctx);
251 }
252 
253 #define CACHE_PTR_INCREMENT (CACHE_LINE_SIZE/sizeof(void*))
254 
255 #define LINK_ACTIVE(ctx) ((ctx)->ifc_link_state == LINK_STATE_UP)
256 #define CTX_IS_VF(ctx) ((ctx)->ifc_sctx->isc_flags & IFLIB_IS_VF)
257 
258 #define RX_SW_DESC_MAP_CREATED	(1 << 0)
259 #define TX_SW_DESC_MAP_CREATED	(1 << 1)
260 #define RX_SW_DESC_INUSE        (1 << 3)
261 #define TX_SW_DESC_MAPPED       (1 << 4)
262 
263 typedef struct iflib_sw_rx_desc {
264 	bus_dmamap_t    ifsd_map;         /* bus_dma map for packet */
265 	struct mbuf    *ifsd_m;           /* rx: uninitialized mbuf */
266 	caddr_t         ifsd_cl;          /* direct cluster pointer for rx */
267 	uint16_t	ifsd_flags;
268 } *iflib_rxsd_t;
269 
270 typedef struct iflib_sw_tx_desc_val {
271 	bus_dmamap_t    ifsd_map;         /* bus_dma map for packet */
272 	struct mbuf    *ifsd_m;           /* pkthdr mbuf */
273 	uint8_t		ifsd_flags;
274 } *iflib_txsd_val_t;
275 
276 typedef struct iflib_sw_tx_desc_array {
277 	bus_dmamap_t    *ifsd_map;         /* bus_dma maps for packet */
278 	struct mbuf    **ifsd_m;           /* pkthdr mbufs */
279 	uint8_t		*ifsd_flags;
280 } iflib_txsd_array_t;
281 
282 
283 /* magic number that should be high enough for any hardware */
284 #define IFLIB_MAX_TX_SEGS		128
285 #define IFLIB_MAX_RX_SEGS		32
286 #define IFLIB_RX_COPY_THRESH		128
287 #define IFLIB_MAX_RX_REFRESH		32
288 #define IFLIB_QUEUE_IDLE		0
289 #define IFLIB_QUEUE_HUNG		1
290 #define IFLIB_QUEUE_WORKING		2
291 
292 /* this should really scale with ring size - 32 is a fairly arbitrary value for this */
293 #define TX_BATCH_SIZE			16
294 
295 #define IFLIB_RESTART_BUDGET		8
296 
297 #define	IFC_LEGACY		0x1
298 #define	IFC_QFLUSH		0x2
299 #define	IFC_MULTISEG		0x4
300 #define	IFC_DMAR		0x8
301 
302 #define CSUM_OFFLOAD		(CSUM_IP_TSO|CSUM_IP6_TSO|CSUM_IP| \
303 				 CSUM_IP_UDP|CSUM_IP_TCP|CSUM_IP_SCTP| \
304 				 CSUM_IP6_UDP|CSUM_IP6_TCP|CSUM_IP6_SCTP)
305 struct iflib_txq {
306 	uint16_t	ift_in_use;
307 	uint16_t	ift_cidx;
308 	uint16_t	ift_cidx_processed;
309 	uint16_t	ift_pidx;
310 	uint8_t		ift_gen;
311 	uint8_t		ift_db_pending;
312 	uint8_t		ift_db_pending_queued;
313 	uint8_t		ift_npending;
314 	/* implicit pad */
315 	uint64_t	ift_processed;
316 	uint64_t	ift_cleaned;
317 #if MEMORY_LOGGING
318 	uint64_t	ift_enqueued;
319 	uint64_t	ift_dequeued;
320 #endif
321 	uint64_t	ift_no_tx_dma_setup;
322 	uint64_t	ift_no_desc_avail;
323 	uint64_t	ift_mbuf_defrag_failed;
324 	uint64_t	ift_mbuf_defrag;
325 	uint64_t	ift_map_failed;
326 	uint64_t	ift_txd_encap_efbig;
327 	uint64_t	ift_pullups;
328 
329 	struct mtx	ift_mtx;
330 	struct mtx	ift_db_mtx;
331 
332 	/* constant values */
333 	if_ctx_t	ift_ctx;
334 	struct ifmp_ring        **ift_br;
335 	struct grouptask	ift_task;
336 	uint16_t	ift_size;
337 	uint16_t	ift_id;
338 	struct callout	ift_timer;
339 	struct callout	ift_db_check;
340 
341 	iflib_txsd_array_t	ift_sds;
342 	uint8_t			ift_nbr;
343 	uint8_t			ift_qstatus;
344 	uint8_t			ift_active;
345 	uint8_t			ift_closed;
346 	int			ift_watchdog_time;
347 	struct iflib_filter_info ift_filter_info;
348 	bus_dma_tag_t		ift_desc_tag;
349 	bus_dma_tag_t		ift_tso_desc_tag;
350 	iflib_dma_info_t	ift_ifdi;
351 #define MTX_NAME_LEN 16
352 	char                    ift_mtx_name[MTX_NAME_LEN];
353 	char                    ift_db_mtx_name[MTX_NAME_LEN];
354 	bus_dma_segment_t	ift_segs[IFLIB_MAX_TX_SEGS]  __aligned(CACHE_LINE_SIZE);
355 } __aligned(CACHE_LINE_SIZE);
356 
357 struct iflib_fl {
358 	uint16_t	ifl_cidx;
359 	uint16_t	ifl_pidx;
360 	uint16_t	ifl_credits;
361 	uint8_t		ifl_gen;
362 #if MEMORY_LOGGING
363 	uint64_t	ifl_m_enqueued;
364 	uint64_t	ifl_m_dequeued;
365 	uint64_t	ifl_cl_enqueued;
366 	uint64_t	ifl_cl_dequeued;
367 #endif
368 	/* implicit pad */
369 
370 	/* constant */
371 	uint16_t	ifl_size;
372 	uint16_t	ifl_buf_size;
373 	uint16_t	ifl_cltype;
374 	uma_zone_t	ifl_zone;
375 	iflib_rxsd_t	ifl_sds;
376 	iflib_rxq_t	ifl_rxq;
377 	uint8_t		ifl_id;
378 	bus_dma_tag_t           ifl_desc_tag;
379 	iflib_dma_info_t	ifl_ifdi;
380 	uint64_t	ifl_bus_addrs[IFLIB_MAX_RX_REFRESH] __aligned(CACHE_LINE_SIZE);
381 	caddr_t		ifl_vm_addrs[IFLIB_MAX_RX_REFRESH];
382 }  __aligned(CACHE_LINE_SIZE);
383 
384 static inline int
385 get_inuse(int size, int cidx, int pidx, int gen)
386 {
387 	int used;
388 
389 	if (pidx > cidx)
390 		used = pidx - cidx;
391 	else if (pidx < cidx)
392 		used = size - cidx + pidx;
393 	else if (gen == 0 && pidx == cidx)
394 		used = 0;
395 	else if (gen == 1 && pidx == cidx)
396 		used = size;
397 	else
398 		panic("bad state");
399 
400 	return (used);
401 }
402 
403 #define TXQ_AVAIL(txq) (txq->ift_size - get_inuse(txq->ift_size, txq->ift_cidx, txq->ift_pidx, txq->ift_gen))
404 
405 #define IDXDIFF(head, tail, wrap) \
406 	((head) >= (tail) ? (head) - (tail) : (wrap) - (tail) + (head))
407 
408 struct iflib_rxq {
409 	/* If there is a separate completion queue -
410 	 * these are the cq cidx and pidx. Otherwise
411 	 * these are unused.
412 	 */
413 	uint16_t	ifr_size;
414 	uint16_t	ifr_cq_cidx;
415 	uint16_t	ifr_cq_pidx;
416 	uint8_t		ifr_cq_gen;
417 
418 	if_ctx_t	ifr_ctx;
419 	iflib_fl_t	ifr_fl;
420 	uint64_t	ifr_rx_irq;
421 	uint16_t	ifr_id;
422 	uint8_t		ifr_lro_enabled;
423 	uint8_t		ifr_nfl;
424 	struct lro_ctrl			ifr_lc;
425 	struct grouptask        ifr_task;
426 	struct iflib_filter_info ifr_filter_info;
427 	iflib_dma_info_t		ifr_ifdi;
428 	/* dynamically allocate if any drivers need a value substantially larger than this */
429 	struct if_rxd_frag	ifr_frags[IFLIB_MAX_RX_SEGS] __aligned(CACHE_LINE_SIZE);
430 }  __aligned(CACHE_LINE_SIZE);
431 
432 /*
433  * Only allow a single packet to take up most 1/nth of the tx ring
434  */
435 #define MAX_SINGLE_PACKET_FRACTION 12
436 #define IF_BAD_DMA (bus_addr_t)-1
437 
438 static int enable_msix = 1;
439 
440 #define mtx_held(m)	(((m)->mtx_lock & ~MTX_FLAGMASK) != (uintptr_t)0)
441 
442 
443 
444 #define CTX_ACTIVE(ctx) ((if_getdrvflags((ctx)->ifc_ifp) & IFF_DRV_RUNNING))
445 
446 #define CTX_LOCK_INIT(_sc, _name)  mtx_init(&(_sc)->ifc_mtx, _name, "iflib ctx lock", MTX_DEF)
447 
448 #define CTX_LOCK(ctx) mtx_lock(&(ctx)->ifc_mtx)
449 #define CTX_UNLOCK(ctx) mtx_unlock(&(ctx)->ifc_mtx)
450 #define CTX_LOCK_DESTROY(ctx) mtx_destroy(&(ctx)->ifc_mtx)
451 
452 
453 #define TXDB_LOCK_INIT(txq)  mtx_init(&(txq)->ift_db_mtx, (txq)->ift_db_mtx_name, NULL, MTX_DEF)
454 #define TXDB_TRYLOCK(txq) mtx_trylock(&(txq)->ift_db_mtx)
455 #define TXDB_LOCK(txq) mtx_lock(&(txq)->ift_db_mtx)
456 #define TXDB_UNLOCK(txq) mtx_unlock(&(txq)->ift_db_mtx)
457 #define TXDB_LOCK_DESTROY(txq) mtx_destroy(&(txq)->ift_db_mtx)
458 
459 #define CALLOUT_LOCK(txq)	mtx_lock(&txq->ift_mtx)
460 #define CALLOUT_UNLOCK(txq) 	mtx_unlock(&txq->ift_mtx)
461 
462 
463 /* Our boot-time initialization hook */
464 static int	iflib_module_event_handler(module_t, int, void *);
465 
466 static moduledata_t iflib_moduledata = {
467 	"iflib",
468 	iflib_module_event_handler,
469 	NULL
470 };
471 
472 DECLARE_MODULE(iflib, iflib_moduledata, SI_SUB_INIT_IF, SI_ORDER_ANY);
473 MODULE_VERSION(iflib, 1);
474 
475 MODULE_DEPEND(iflib, pci, 1, 1, 1);
476 MODULE_DEPEND(iflib, ether, 1, 1, 1);
477 
478 TASKQGROUP_DEFINE(if_io_tqg, mp_ncpus, 1);
479 TASKQGROUP_DEFINE(if_config_tqg, 1, 1);
480 
481 #ifndef IFLIB_DEBUG_COUNTERS
482 #ifdef INVARIANTS
483 #define IFLIB_DEBUG_COUNTERS 1
484 #else
485 #define IFLIB_DEBUG_COUNTERS 0
486 #endif /* !INVARIANTS */
487 #endif
488 
489 static SYSCTL_NODE(_net, OID_AUTO, iflib, CTLFLAG_RD, 0,
490                    "iflib driver parameters");
491 
492 /*
493  * XXX need to ensure that this can't accidentally cause the head to be moved backwards
494  */
495 static int iflib_min_tx_latency = 0;
496 
497 SYSCTL_INT(_net_iflib, OID_AUTO, min_tx_latency, CTLFLAG_RW,
498 		   &iflib_min_tx_latency, 0, "minimize transmit latency at the possibel expense of throughput");
499 
500 
501 #if IFLIB_DEBUG_COUNTERS
502 
503 static int iflib_tx_seen;
504 static int iflib_tx_sent;
505 static int iflib_tx_encap;
506 static int iflib_rx_allocs;
507 static int iflib_fl_refills;
508 static int iflib_fl_refills_large;
509 static int iflib_tx_frees;
510 
511 SYSCTL_INT(_net_iflib, OID_AUTO, tx_seen, CTLFLAG_RD,
512 		   &iflib_tx_seen, 0, "# tx mbufs seen");
513 SYSCTL_INT(_net_iflib, OID_AUTO, tx_sent, CTLFLAG_RD,
514 		   &iflib_tx_sent, 0, "# tx mbufs sent");
515 SYSCTL_INT(_net_iflib, OID_AUTO, tx_encap, CTLFLAG_RD,
516 		   &iflib_tx_encap, 0, "# tx mbufs encapped");
517 SYSCTL_INT(_net_iflib, OID_AUTO, tx_frees, CTLFLAG_RD,
518 		   &iflib_tx_frees, 0, "# tx frees");
519 SYSCTL_INT(_net_iflib, OID_AUTO, rx_allocs, CTLFLAG_RD,
520 		   &iflib_rx_allocs, 0, "# rx allocations");
521 SYSCTL_INT(_net_iflib, OID_AUTO, fl_refills, CTLFLAG_RD,
522 		   &iflib_fl_refills, 0, "# refills");
523 SYSCTL_INT(_net_iflib, OID_AUTO, fl_refills_large, CTLFLAG_RD,
524 		   &iflib_fl_refills_large, 0, "# large refills");
525 
526 
527 static int iflib_txq_drain_flushing;
528 static int iflib_txq_drain_oactive;
529 static int iflib_txq_drain_notready;
530 static int iflib_txq_drain_encapfail;
531 
532 SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_flushing, CTLFLAG_RD,
533 		   &iflib_txq_drain_flushing, 0, "# drain flushes");
534 SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_oactive, CTLFLAG_RD,
535 		   &iflib_txq_drain_oactive, 0, "# drain oactives");
536 SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_notready, CTLFLAG_RD,
537 		   &iflib_txq_drain_notready, 0, "# drain notready");
538 SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_encapfail, CTLFLAG_RD,
539 		   &iflib_txq_drain_encapfail, 0, "# drain encap fails");
540 
541 
542 static int iflib_encap_load_mbuf_fail;
543 static int iflib_encap_txq_avail_fail;
544 static int iflib_encap_txd_encap_fail;
545 
546 SYSCTL_INT(_net_iflib, OID_AUTO, encap_load_mbuf_fail, CTLFLAG_RD,
547 		   &iflib_encap_load_mbuf_fail, 0, "# busdma load failures");
548 SYSCTL_INT(_net_iflib, OID_AUTO, encap_txq_avail_fail, CTLFLAG_RD,
549 		   &iflib_encap_txq_avail_fail, 0, "# txq avail failures");
550 SYSCTL_INT(_net_iflib, OID_AUTO, encap_txd_encap_fail, CTLFLAG_RD,
551 		   &iflib_encap_txd_encap_fail, 0, "# driver encap failures");
552 
553 static int iflib_task_fn_rxs;
554 static int iflib_rx_intr_enables;
555 static int iflib_fast_intrs;
556 static int iflib_intr_link;
557 static int iflib_intr_msix;
558 static int iflib_rx_unavail;
559 static int iflib_rx_ctx_inactive;
560 static int iflib_rx_zero_len;
561 static int iflib_rx_if_input;
562 static int iflib_rx_mbuf_null;
563 static int iflib_rxd_flush;
564 
565 static int iflib_verbose_debug;
566 
567 SYSCTL_INT(_net_iflib, OID_AUTO, intr_link, CTLFLAG_RD,
568 		   &iflib_intr_link, 0, "# intr link calls");
569 SYSCTL_INT(_net_iflib, OID_AUTO, intr_msix, CTLFLAG_RD,
570 		   &iflib_intr_msix, 0, "# intr msix calls");
571 SYSCTL_INT(_net_iflib, OID_AUTO, task_fn_rx, CTLFLAG_RD,
572 		   &iflib_task_fn_rxs, 0, "# task_fn_rx calls");
573 SYSCTL_INT(_net_iflib, OID_AUTO, rx_intr_enables, CTLFLAG_RD,
574 		   &iflib_rx_intr_enables, 0, "# rx intr enables");
575 SYSCTL_INT(_net_iflib, OID_AUTO, fast_intrs, CTLFLAG_RD,
576 		   &iflib_fast_intrs, 0, "# fast_intr calls");
577 SYSCTL_INT(_net_iflib, OID_AUTO, rx_unavail, CTLFLAG_RD,
578 		   &iflib_rx_unavail, 0, "# times rxeof called with no available data");
579 SYSCTL_INT(_net_iflib, OID_AUTO, rx_ctx_inactive, CTLFLAG_RD,
580 		   &iflib_rx_ctx_inactive, 0, "# times rxeof called with inactive context");
581 SYSCTL_INT(_net_iflib, OID_AUTO, rx_zero_len, CTLFLAG_RD,
582 		   &iflib_rx_zero_len, 0, "# times rxeof saw zero len mbuf");
583 SYSCTL_INT(_net_iflib, OID_AUTO, rx_if_input, CTLFLAG_RD,
584 		   &iflib_rx_if_input, 0, "# times rxeof called if_input");
585 SYSCTL_INT(_net_iflib, OID_AUTO, rx_mbuf_null, CTLFLAG_RD,
586 		   &iflib_rx_mbuf_null, 0, "# times rxeof got null mbuf");
587 SYSCTL_INT(_net_iflib, OID_AUTO, rxd_flush, CTLFLAG_RD,
588 	         &iflib_rxd_flush, 0, "# times rxd_flush called");
589 SYSCTL_INT(_net_iflib, OID_AUTO, verbose_debug, CTLFLAG_RW,
590 		   &iflib_verbose_debug, 0, "enable verbose debugging");
591 
592 #define DBG_COUNTER_INC(name) atomic_add_int(&(iflib_ ## name), 1)
593 
594 #else
595 #define DBG_COUNTER_INC(name)
596 
597 #endif
598 
599 
600 
601 #define IFLIB_DEBUG 0
602 
603 static void iflib_tx_structures_free(if_ctx_t ctx);
604 static void iflib_rx_structures_free(if_ctx_t ctx);
605 static int iflib_queues_alloc(if_ctx_t ctx);
606 static int iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq);
607 static int iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, int cidx);
608 static int iflib_qset_structures_setup(if_ctx_t ctx);
609 static int iflib_msix_init(if_ctx_t ctx);
610 static int iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filterarg, int *rid, char *str);
611 static void iflib_txq_check_drain(iflib_txq_t txq, int budget);
612 static uint32_t iflib_txq_can_drain(struct ifmp_ring *);
613 static int iflib_register(if_ctx_t);
614 static void iflib_init_locked(if_ctx_t ctx);
615 static void iflib_add_device_sysctl_pre(if_ctx_t ctx);
616 static void iflib_add_device_sysctl_post(if_ctx_t ctx);
617 
618 
619 #ifdef DEV_NETMAP
620 #include <sys/selinfo.h>
621 #include <net/netmap.h>
622 #include <dev/netmap/netmap_kern.h>
623 
624 MODULE_DEPEND(iflib, netmap, 1, 1, 1);
625 
626 /*
627  * device-specific sysctl variables:
628  *
629  * ixl_crcstrip: 0: keep CRC in rx frames (default), 1: strip it.
630  *	During regular operations the CRC is stripped, but on some
631  *	hardware reception of frames not multiple of 64 is slower,
632  *	so using crcstrip=0 helps in benchmarks.
633  *
634  * ixl_rx_miss, ixl_rx_miss_bufs:
635  *	count packets that might be missed due to lost interrupts.
636  */
637 SYSCTL_DECL(_dev_netmap);
638 /*
639  * The xl driver by default strips CRCs and we do not override it.
640  */
641 
642 int iflib_crcstrip = 1;
643 SYSCTL_INT(_dev_netmap, OID_AUTO, iflib_crcstrip,
644     CTLFLAG_RW, &iflib_crcstrip, 1, "strip CRC on rx frames");
645 
646 int iflib_rx_miss, iflib_rx_miss_bufs;
647 SYSCTL_INT(_dev_netmap, OID_AUTO, iflib_rx_miss,
648     CTLFLAG_RW, &iflib_rx_miss, 0, "potentially missed rx intr");
649 SYSCTL_INT(_dev_netmap, OID_AUTO, ixl_rx_miss_bufs,
650     CTLFLAG_RW, &iflib_rx_miss_bufs, 0, "potentially missed rx intr bufs");
651 
652 /*
653  * Register/unregister. We are already under netmap lock.
654  * Only called on the first register or the last unregister.
655  */
656 static int
657 iflib_netmap_register(struct netmap_adapter *na, int onoff)
658 {
659 	struct ifnet *ifp = na->ifp;
660 	if_ctx_t ctx = ifp->if_softc;
661 
662 	CTX_LOCK(ctx);
663 	IFDI_INTR_DISABLE(ctx);
664 
665 	/* Tell the stack that the interface is no longer active */
666 	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
667 
668 	if (!CTX_IS_VF(ctx))
669 		IFDI_CRCSTRIP_SET(ctx, onoff);
670 
671 	/* enable or disable flags and callbacks in na and ifp */
672 	if (onoff) {
673 		nm_set_native_flags(na);
674 	} else {
675 		nm_clear_native_flags(na);
676 	}
677 	IFDI_INIT(ctx);
678 	IFDI_CRCSTRIP_SET(ctx, onoff); // XXX why twice ?
679 	CTX_UNLOCK(ctx);
680 	return (ifp->if_drv_flags & IFF_DRV_RUNNING ? 0 : 1);
681 }
682 
683 /*
684  * Reconcile kernel and user view of the transmit ring.
685  *
686  * All information is in the kring.
687  * Userspace wants to send packets up to the one before kring->rhead,
688  * kernel knows kring->nr_hwcur is the first unsent packet.
689  *
690  * Here we push packets out (as many as possible), and possibly
691  * reclaim buffers from previously completed transmission.
692  *
693  * The caller (netmap) guarantees that there is only one instance
694  * running at any time. Any interference with other driver
695  * methods should be handled by the individual drivers.
696  */
697 static int
698 iflib_netmap_txsync(struct netmap_kring *kring, int flags)
699 {
700 	struct netmap_adapter *na = kring->na;
701 	struct ifnet *ifp = na->ifp;
702 	struct netmap_ring *ring = kring->ring;
703 	u_int nm_i;	/* index into the netmap ring */
704 	u_int nic_i;	/* index into the NIC ring */
705 	u_int n;
706 	u_int const lim = kring->nkr_num_slots - 1;
707 	u_int const head = kring->rhead;
708 	struct if_pkt_info pi;
709 
710 	/*
711 	 * interrupts on every tx packet are expensive so request
712 	 * them every half ring, or where NS_REPORT is set
713 	 */
714 	u_int report_frequency = kring->nkr_num_slots >> 1;
715 	/* device-specific */
716 	if_ctx_t ctx = ifp->if_softc;
717 	iflib_txq_t txq = &ctx->ifc_txqs[kring->ring_id];
718 
719 	pi.ipi_segs = txq->ift_segs;
720 	pi.ipi_qsidx = kring->ring_id;
721 	pi.ipi_ndescs = 0;
722 
723 	bus_dmamap_sync(txq->ift_desc_tag, txq->ift_ifdi->idi_map,
724 					BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
725 
726 
727 	/*
728 	 * First part: process new packets to send.
729 	 * nm_i is the current index in the netmap ring,
730 	 * nic_i is the corresponding index in the NIC ring.
731 	 *
732 	 * If we have packets to send (nm_i != head)
733 	 * iterate over the netmap ring, fetch length and update
734 	 * the corresponding slot in the NIC ring. Some drivers also
735 	 * need to update the buffer's physical address in the NIC slot
736 	 * even NS_BUF_CHANGED is not set (PNMB computes the addresses).
737 	 *
738 	 * The netmap_reload_map() calls is especially expensive,
739 	 * even when (as in this case) the tag is 0, so do only
740 	 * when the buffer has actually changed.
741 	 *
742 	 * If possible do not set the report/intr bit on all slots,
743 	 * but only a few times per ring or when NS_REPORT is set.
744 	 *
745 	 * Finally, on 10G and faster drivers, it might be useful
746 	 * to prefetch the next slot and txr entry.
747 	 */
748 
749 	nm_i = kring->nr_hwcur;
750 	if (nm_i != head) {	/* we have new packets to send */
751 		nic_i = netmap_idx_k2n(kring, nm_i);
752 
753 		__builtin_prefetch(&ring->slot[nm_i]);
754 		__builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i]);
755 		__builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i]);
756 
757 		for (n = 0; nm_i != head; n++) {
758 			struct netmap_slot *slot = &ring->slot[nm_i];
759 			u_int len = slot->len;
760 			uint64_t paddr;
761 			void *addr = PNMB(na, slot, &paddr);
762 			int flags = (slot->flags & NS_REPORT ||
763 				nic_i == 0 || nic_i == report_frequency) ?
764 				IPI_TX_INTR : 0;
765 
766 			/* device-specific */
767 			pi.ipi_pidx = nic_i;
768 			pi.ipi_flags = flags;
769 
770 			/* Fill the slot in the NIC ring. */
771 			ctx->isc_txd_encap(ctx->ifc_softc, &pi);
772 
773 			/* prefetch for next round */
774 			__builtin_prefetch(&ring->slot[nm_i + 1]);
775 			__builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i + 1]);
776 			__builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i + 1]);
777 
778 			NM_CHECK_ADDR_LEN(na, addr, len);
779 
780 			if (slot->flags & NS_BUF_CHANGED) {
781 				/* buffer has changed, reload map */
782 				netmap_reload_map(na, txq->ift_desc_tag, txq->ift_sds.ifsd_map[nic_i], addr);
783 			}
784 			slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED);
785 
786 			/* make sure changes to the buffer are synced */
787 			bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_sds.ifsd_map[nic_i],
788 							BUS_DMASYNC_PREWRITE);
789 
790 			nm_i = nm_next(nm_i, lim);
791 			nic_i = nm_next(nic_i, lim);
792 		}
793 		kring->nr_hwcur = head;
794 
795 		/* synchronize the NIC ring */
796 		bus_dmamap_sync(txq->ift_desc_tag, txq->ift_ifdi->idi_map,
797 						BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
798 
799 		/* (re)start the tx unit up to slot nic_i (excluded) */
800 		ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, nic_i);
801 	}
802 
803 	/*
804 	 * Second part: reclaim buffers for completed transmissions.
805 	 */
806 	if (iflib_tx_credits_update(ctx, txq)) {
807 		/* some tx completed, increment avail */
808 		nic_i = txq->ift_cidx_processed;
809 		kring->nr_hwtail = nm_prev(netmap_idx_n2k(kring, nic_i), lim);
810 	}
811 	return (0);
812 }
813 
814 /*
815  * Reconcile kernel and user view of the receive ring.
816  * Same as for the txsync, this routine must be efficient.
817  * The caller guarantees a single invocations, but races against
818  * the rest of the driver should be handled here.
819  *
820  * On call, kring->rhead is the first packet that userspace wants
821  * to keep, and kring->rcur is the wakeup point.
822  * The kernel has previously reported packets up to kring->rtail.
823  *
824  * If (flags & NAF_FORCE_READ) also check for incoming packets irrespective
825  * of whether or not we received an interrupt.
826  */
827 static int
828 iflib_netmap_rxsync(struct netmap_kring *kring, int flags)
829 {
830 	struct netmap_adapter *na = kring->na;
831 	struct ifnet *ifp = na->ifp;
832 	struct netmap_ring *ring = kring->ring;
833 	u_int nm_i;	/* index into the netmap ring */
834 	u_int nic_i;	/* index into the NIC ring */
835 	u_int i, n;
836 	u_int const lim = kring->nkr_num_slots - 1;
837 	u_int const head = kring->rhead;
838 	int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;
839 	struct if_rxd_info ri;
840 	/* device-specific */
841 	if_ctx_t ctx = ifp->if_softc;
842 	iflib_rxq_t rxq = &ctx->ifc_rxqs[kring->ring_id];
843 	iflib_fl_t fl = rxq->ifr_fl;
844 	if (head > lim)
845 		return netmap_ring_reinit(kring);
846 
847 	bzero(&ri, sizeof(ri));
848 	ri.iri_qsidx = kring->ring_id;
849 	ri.iri_ifp = ctx->ifc_ifp;
850 	/* XXX check sync modes */
851 	for (i = 0, fl = rxq->ifr_fl; i < rxq->ifr_nfl; i++, fl++)
852 		bus_dmamap_sync(rxq->ifr_fl[i].ifl_desc_tag, fl->ifl_ifdi->idi_map,
853 				BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
854 
855 	/*
856 	 * First part: import newly received packets.
857 	 *
858 	 * nm_i is the index of the next free slot in the netmap ring,
859 	 * nic_i is the index of the next received packet in the NIC ring,
860 	 * and they may differ in case if_init() has been called while
861 	 * in netmap mode. For the receive ring we have
862 	 *
863 	 *	nic_i = rxr->next_check;
864 	 *	nm_i = kring->nr_hwtail (previous)
865 	 * and
866 	 *	nm_i == (nic_i + kring->nkr_hwofs) % ring_size
867 	 *
868 	 * rxr->next_check is set to 0 on a ring reinit
869 	 */
870 	if (netmap_no_pendintr || force_update) {
871 		int crclen = iflib_crcstrip ? 0 : 4;
872 		int error, avail;
873 		uint16_t slot_flags = kring->nkr_slot_flags;
874 
875 		for (fl = rxq->ifr_fl, i = 0; i < rxq->ifr_nfl; i++, fl++) {
876 			nic_i = fl->ifl_cidx;
877 			nm_i = netmap_idx_n2k(kring, nic_i);
878 			avail = ctx->isc_rxd_available(ctx->ifc_softc, kring->ring_id, nic_i);
879 			for (n = 0; avail > 0; n++, avail--) {
880 				error = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri);
881 				if (error)
882 					ring->slot[nm_i].len = 0;
883 				else
884 					ring->slot[nm_i].len = ri.iri_len - crclen;
885 				ring->slot[nm_i].flags = slot_flags;
886 				bus_dmamap_sync(fl->ifl_ifdi->idi_tag,
887 								fl->ifl_sds[nic_i].ifsd_map, BUS_DMASYNC_POSTREAD);
888 				nm_i = nm_next(nm_i, lim);
889 				nic_i = nm_next(nic_i, lim);
890 			}
891 			if (n) { /* update the state variables */
892 				if (netmap_no_pendintr && !force_update) {
893 					/* diagnostics */
894 					iflib_rx_miss ++;
895 					iflib_rx_miss_bufs += n;
896 				}
897 				fl->ifl_cidx = nic_i;
898 				kring->nr_hwtail = nm_i;
899 			}
900 			kring->nr_kflags &= ~NKR_PENDINTR;
901 		}
902 	}
903 	/*
904 	 * Second part: skip past packets that userspace has released.
905 	 * (kring->nr_hwcur to head excluded),
906 	 * and make the buffers available for reception.
907 	 * As usual nm_i is the index in the netmap ring,
908 	 * nic_i is the index in the NIC ring, and
909 	 * nm_i == (nic_i + kring->nkr_hwofs) % ring_size
910 	 */
911 	/* XXX not sure how this will work with multiple free lists */
912 	nm_i = kring->nr_hwcur;
913 	if (nm_i != head) {
914 		nic_i = netmap_idx_k2n(kring, nm_i);
915 		for (n = 0; nm_i != head; n++) {
916 			struct netmap_slot *slot = &ring->slot[nm_i];
917 			uint64_t paddr;
918 			caddr_t vaddr;
919 			void *addr = PNMB(na, slot, &paddr);
920 
921 			if (addr == NETMAP_BUF_BASE(na)) /* bad buf */
922 				goto ring_reset;
923 
924 			vaddr = addr;
925 			if (slot->flags & NS_BUF_CHANGED) {
926 				/* buffer has changed, reload map */
927 				netmap_reload_map(na, fl->ifl_ifdi->idi_tag, fl->ifl_sds[nic_i].ifsd_map, addr);
928 				slot->flags &= ~NS_BUF_CHANGED;
929 			}
930 			/*
931 			 * XXX we should be batching this operation - TODO
932 			 */
933 			ctx->isc_rxd_refill(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, nic_i, &paddr, &vaddr, 1);
934 			bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_sds[nic_i].ifsd_map,
935 			    BUS_DMASYNC_PREREAD);
936 			nm_i = nm_next(nm_i, lim);
937 			nic_i = nm_next(nic_i, lim);
938 		}
939 		kring->nr_hwcur = head;
940 
941 		bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
942 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
943 		/*
944 		 * IMPORTANT: we must leave one free slot in the ring,
945 		 * so move nic_i back by one unit
946 		 */
947 		nic_i = nm_prev(nic_i, lim);
948 		ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, nic_i);
949 	}
950 
951 	return 0;
952 
953 ring_reset:
954 	return netmap_ring_reinit(kring);
955 }
956 
957 static int
958 iflib_netmap_attach(if_ctx_t ctx)
959 {
960 	struct netmap_adapter na;
961 
962 	bzero(&na, sizeof(na));
963 
964 	na.ifp = ctx->ifc_ifp;
965 	na.na_flags = NAF_BDG_MAYSLEEP;
966 	MPASS(ctx->ifc_softc_ctx.isc_ntxqsets);
967 	MPASS(ctx->ifc_softc_ctx.isc_nrxqsets);
968 
969 	na.num_tx_desc = ctx->ifc_sctx->isc_ntxd;
970 	na.num_rx_desc = ctx->ifc_sctx->isc_ntxd;
971 	na.nm_txsync = iflib_netmap_txsync;
972 	na.nm_rxsync = iflib_netmap_rxsync;
973 	na.nm_register = iflib_netmap_register;
974 	na.num_tx_rings = ctx->ifc_softc_ctx.isc_ntxqsets;
975 	na.num_rx_rings = ctx->ifc_softc_ctx.isc_nrxqsets;
976 	return (netmap_attach(&na));
977 }
978 
979 static void
980 iflib_netmap_txq_init(if_ctx_t ctx, iflib_txq_t txq)
981 {
982 	struct netmap_adapter *na = NA(ctx->ifc_ifp);
983 	struct netmap_slot *slot;
984 
985 	slot = netmap_reset(na, NR_TX, txq->ift_id, 0);
986 	if (slot == 0)
987 		return;
988 
989 	for (int i = 0; i < ctx->ifc_sctx->isc_ntxd; i++) {
990 
991 		/*
992 		 * In netmap mode, set the map for the packet buffer.
993 		 * NOTE: Some drivers (not this one) also need to set
994 		 * the physical buffer address in the NIC ring.
995 		 * netmap_idx_n2k() maps a nic index, i, into the corresponding
996 		 * netmap slot index, si
997 		 */
998 		int si = netmap_idx_n2k(&na->tx_rings[txq->ift_id], i);
999 		netmap_load_map(na, txq->ift_desc_tag, txq->ift_sds.ifsd_map[i], NMB(na, slot + si));
1000 	}
1001 }
1002 static void
1003 iflib_netmap_rxq_init(if_ctx_t ctx, iflib_rxq_t rxq)
1004 {
1005 	struct netmap_adapter *na = NA(ctx->ifc_ifp);
1006 	struct netmap_slot *slot;
1007 	iflib_rxsd_t sd;
1008 	int nrxd;
1009 
1010 	slot = netmap_reset(na, NR_RX, rxq->ifr_id, 0);
1011 	if (slot == 0)
1012 		return;
1013 	sd = rxq->ifr_fl[0].ifl_sds;
1014 	nrxd = ctx->ifc_sctx->isc_nrxd;
1015 	for (int i = 0; i < nrxd; i++, sd++) {
1016 			int sj = netmap_idx_n2k(&na->rx_rings[rxq->ifr_id], i);
1017 			uint64_t paddr;
1018 			void *addr;
1019 			caddr_t vaddr;
1020 
1021 			vaddr = addr = PNMB(na, slot + sj, &paddr);
1022 			netmap_load_map(na, rxq->ifr_fl[0].ifl_ifdi->idi_tag, sd->ifsd_map, addr);
1023 			/* Update descriptor and the cached value */
1024 			ctx->isc_rxd_refill(ctx->ifc_softc, rxq->ifr_id, 0 /* fl_id */, i, &paddr, &vaddr, 1);
1025 	}
1026 	/* preserve queue */
1027 	if (ctx->ifc_ifp->if_capenable & IFCAP_NETMAP) {
1028 		struct netmap_kring *kring = &na->rx_rings[rxq->ifr_id];
1029 		int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
1030 		ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, 0 /* fl_id */, t);
1031 	} else
1032 		ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, 0 /* fl_id */, nrxd-1);
1033 }
1034 
1035 #define iflib_netmap_detach(ifp) netmap_detach(ifp)
1036 
1037 #else
1038 #define iflib_netmap_txq_init(ctx, txq)
1039 #define iflib_netmap_rxq_init(ctx, rxq)
1040 #define iflib_netmap_detach(ifp)
1041 
1042 #define iflib_netmap_attach(ctx) (0)
1043 #define netmap_rx_irq(ifp, qid, budget) (0)
1044 
1045 #endif
1046 
1047 #if defined(__i386__) || defined(__amd64__)
1048 static __inline void
1049 prefetch(void *x)
1050 {
1051 	__asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x));
1052 }
1053 #else
1054 #define prefetch(x)
1055 #endif
1056 
1057 static void
1058 _iflib_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err)
1059 {
1060 	if (err)
1061 		return;
1062 	*(bus_addr_t *) arg = segs[0].ds_addr;
1063 }
1064 
1065 int
1066 iflib_dma_alloc(if_ctx_t ctx, int size, iflib_dma_info_t dma, int mapflags)
1067 {
1068 	int err;
1069 	if_shared_ctx_t sctx = ctx->ifc_sctx;
1070 	device_t dev = ctx->ifc_dev;
1071 
1072 	KASSERT(sctx->isc_q_align != 0, ("alignment value not initialized"));
1073 
1074 	err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
1075 				sctx->isc_q_align, 0,	/* alignment, bounds */
1076 				BUS_SPACE_MAXADDR,	/* lowaddr */
1077 				BUS_SPACE_MAXADDR,	/* highaddr */
1078 				NULL, NULL,		/* filter, filterarg */
1079 				size,			/* maxsize */
1080 				1,			/* nsegments */
1081 				size,			/* maxsegsize */
1082 				BUS_DMA_ALLOCNOW,	/* flags */
1083 				NULL,			/* lockfunc */
1084 				NULL,			/* lockarg */
1085 				&dma->idi_tag);
1086 	if (err) {
1087 		device_printf(dev,
1088 		    "%s: bus_dma_tag_create failed: %d\n",
1089 		    __func__, err);
1090 		goto fail_0;
1091 	}
1092 
1093 	err = bus_dmamem_alloc(dma->idi_tag, (void**) &dma->idi_vaddr,
1094 	    BUS_DMA_NOWAIT | BUS_DMA_COHERENT | BUS_DMA_ZERO, &dma->idi_map);
1095 	if (err) {
1096 		device_printf(dev,
1097 		    "%s: bus_dmamem_alloc(%ju) failed: %d\n",
1098 		    __func__, (uintmax_t)size, err);
1099 		goto fail_1;
1100 	}
1101 
1102 	dma->idi_paddr = IF_BAD_DMA;
1103 	err = bus_dmamap_load(dma->idi_tag, dma->idi_map, dma->idi_vaddr,
1104 	    size, _iflib_dmamap_cb, &dma->idi_paddr, mapflags | BUS_DMA_NOWAIT);
1105 	if (err || dma->idi_paddr == IF_BAD_DMA) {
1106 		device_printf(dev,
1107 		    "%s: bus_dmamap_load failed: %d\n",
1108 		    __func__, err);
1109 		goto fail_2;
1110 	}
1111 
1112 	dma->idi_size = size;
1113 	return (0);
1114 
1115 fail_2:
1116 	bus_dmamem_free(dma->idi_tag, dma->idi_vaddr, dma->idi_map);
1117 fail_1:
1118 	bus_dma_tag_destroy(dma->idi_tag);
1119 fail_0:
1120 	dma->idi_tag = NULL;
1121 
1122 	return (err);
1123 }
1124 
1125 int
1126 iflib_dma_alloc_multi(if_ctx_t ctx, int *sizes, iflib_dma_info_t *dmalist, int mapflags, int count)
1127 {
1128 	int i, err;
1129 	iflib_dma_info_t *dmaiter;
1130 
1131 	dmaiter = dmalist;
1132 	for (i = 0; i < count; i++, dmaiter++) {
1133 		if ((err = iflib_dma_alloc(ctx, sizes[i], *dmaiter, mapflags)) != 0)
1134 			break;
1135 	}
1136 	if (err)
1137 		iflib_dma_free_multi(dmalist, i);
1138 	return (err);
1139 }
1140 
1141 void
1142 iflib_dma_free(iflib_dma_info_t dma)
1143 {
1144 	if (dma->idi_tag == NULL)
1145 		return;
1146 	if (dma->idi_paddr != IF_BAD_DMA) {
1147 		bus_dmamap_sync(dma->idi_tag, dma->idi_map,
1148 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1149 		bus_dmamap_unload(dma->idi_tag, dma->idi_map);
1150 		dma->idi_paddr = IF_BAD_DMA;
1151 	}
1152 	if (dma->idi_vaddr != NULL) {
1153 		bus_dmamem_free(dma->idi_tag, dma->idi_vaddr, dma->idi_map);
1154 		dma->idi_vaddr = NULL;
1155 	}
1156 	bus_dma_tag_destroy(dma->idi_tag);
1157 	dma->idi_tag = NULL;
1158 }
1159 
1160 void
1161 iflib_dma_free_multi(iflib_dma_info_t *dmalist, int count)
1162 {
1163 	int i;
1164 	iflib_dma_info_t *dmaiter = dmalist;
1165 
1166 	for (i = 0; i < count; i++, dmaiter++)
1167 		iflib_dma_free(*dmaiter);
1168 }
1169 
1170 static int
1171 iflib_fast_intr(void *arg)
1172 {
1173 	iflib_filter_info_t info = arg;
1174 	struct grouptask *gtask = info->ifi_task;
1175 
1176 	DBG_COUNTER_INC(fast_intrs);
1177 	if (info->ifi_filter != NULL && info->ifi_filter(info->ifi_filter_arg) == FILTER_HANDLED)
1178 		return (FILTER_HANDLED);
1179 
1180 	GROUPTASK_ENQUEUE(gtask);
1181 	return (FILTER_HANDLED);
1182 }
1183 
1184 static int
1185 _iflib_irq_alloc(if_ctx_t ctx, if_irq_t irq, int rid,
1186 	driver_filter_t filter, driver_intr_t handler, void *arg,
1187 				 char *name)
1188 {
1189 	int rc;
1190 	struct resource *res;
1191 	void *tag;
1192 	device_t dev = ctx->ifc_dev;
1193 
1194 	MPASS(rid < 512);
1195 	irq->ii_rid = rid;
1196 	res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &irq->ii_rid,
1197 				     RF_SHAREABLE | RF_ACTIVE);
1198 	if (res == NULL) {
1199 		device_printf(dev,
1200 		    "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
1201 		return (ENOMEM);
1202 	}
1203 	irq->ii_res = res;
1204 	KASSERT(filter == NULL || handler == NULL, ("filter and handler can't both be non-NULL"));
1205 	rc = bus_setup_intr(dev, res, INTR_MPSAFE | INTR_TYPE_NET,
1206 						filter, handler, arg, &tag);
1207 	if (rc != 0) {
1208 		device_printf(dev,
1209 		    "failed to setup interrupt for rid %d, name %s: %d\n",
1210 					  rid, name ? name : "unknown", rc);
1211 		return (rc);
1212 	} else if (name)
1213 		bus_describe_intr(dev, res, tag, name);
1214 
1215 	irq->ii_tag = tag;
1216 	return (0);
1217 }
1218 
1219 
1220 /*********************************************************************
1221  *
1222  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
1223  *  the information needed to transmit a packet on the wire. This is
1224  *  called only once at attach, setup is done every reset.
1225  *
1226  **********************************************************************/
1227 
1228 static int
1229 iflib_txsd_alloc(iflib_txq_t txq)
1230 {
1231 	if_ctx_t ctx = txq->ift_ctx;
1232 	if_shared_ctx_t sctx = ctx->ifc_sctx;
1233 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
1234 	device_t dev = ctx->ifc_dev;
1235 	int err, nsegments, ntsosegments;
1236 
1237 	nsegments = scctx->isc_tx_nsegments;
1238 	ntsosegments = scctx->isc_tx_tso_segments_max;
1239 	MPASS(sctx->isc_ntxd > 0);
1240 	MPASS(nsegments > 0);
1241 	MPASS(ntsosegments > 0);
1242 	/*
1243 	 * Setup DMA descriptor areas.
1244 	 */
1245 	if ((err = bus_dma_tag_create(bus_get_dma_tag(dev),
1246 			       1, 0,			/* alignment, bounds */
1247 			       BUS_SPACE_MAXADDR,	/* lowaddr */
1248 			       BUS_SPACE_MAXADDR,	/* highaddr */
1249 			       NULL, NULL,		/* filter, filterarg */
1250 			       sctx->isc_tx_maxsize,		/* maxsize */
1251 			       nsegments,	/* nsegments */
1252 			       sctx->isc_tx_maxsegsize,	/* maxsegsize */
1253 			       0,			/* flags */
1254 			       NULL,			/* lockfunc */
1255 			       NULL,			/* lockfuncarg */
1256 			       &txq->ift_desc_tag))) {
1257 		device_printf(dev,"Unable to allocate TX DMA tag: %d\n", err);
1258 		device_printf(dev,"maxsize: %zd nsegments: %d maxsegsize: %zd\n",
1259 					  sctx->isc_tx_maxsize, nsegments, sctx->isc_tx_maxsegsize);
1260 		goto fail;
1261 	}
1262 #ifdef INVARIANTS
1263 	device_printf(dev,"maxsize: %zd nsegments: %d maxsegsize: %zd\n",
1264 		      sctx->isc_tx_maxsize, nsegments, sctx->isc_tx_maxsegsize);
1265 #endif
1266 	device_printf(dev,"TSO maxsize: %d ntsosegments: %d maxsegsize: %d\n",
1267 		      scctx->isc_tx_tso_size_max, ntsosegments,
1268 		      scctx->isc_tx_tso_segsize_max);
1269 	if ((err = bus_dma_tag_create(bus_get_dma_tag(dev),
1270 			       1, 0,			/* alignment, bounds */
1271 			       BUS_SPACE_MAXADDR,	/* lowaddr */
1272 			       BUS_SPACE_MAXADDR,	/* highaddr */
1273 			       NULL, NULL,		/* filter, filterarg */
1274 			       scctx->isc_tx_tso_size_max,		/* maxsize */
1275 			       ntsosegments,	/* nsegments */
1276 			       scctx->isc_tx_tso_segsize_max,	/* maxsegsize */
1277 			       0,			/* flags */
1278 			       NULL,			/* lockfunc */
1279 			       NULL,			/* lockfuncarg */
1280 			       &txq->ift_tso_desc_tag))) {
1281 		device_printf(dev,"Unable to allocate TX TSO DMA tag: %d\n", err);
1282 
1283 		goto fail;
1284 	}
1285 #ifdef INVARIANTS
1286 	device_printf(dev,"TSO maxsize: %d ntsosegments: %d maxsegsize: %d\n",
1287 		      scctx->isc_tx_tso_size_max, ntsosegments,
1288 		      scctx->isc_tx_tso_segsize_max);
1289 #endif
1290 	if (!(txq->ift_sds.ifsd_flags =
1291 	    (uint8_t *) malloc(sizeof(uint8_t) *
1292 	    sctx->isc_ntxd, M_IFLIB, M_NOWAIT | M_ZERO))) {
1293 		device_printf(dev, "Unable to allocate tx_buffer memory\n");
1294 		err = ENOMEM;
1295 		goto fail;
1296 	}
1297 	if (!(txq->ift_sds.ifsd_m =
1298 	    (struct mbuf **) malloc(sizeof(struct mbuf *) *
1299 	    sctx->isc_ntxd, M_IFLIB, M_NOWAIT | M_ZERO))) {
1300 		device_printf(dev, "Unable to allocate tx_buffer memory\n");
1301 		err = ENOMEM;
1302 		goto fail;
1303 	}
1304 
1305         /* Create the descriptor buffer dma maps */
1306 #if defined(ACPI_DMAR) || (!(defined(__i386__) && !defined(__amd64__)))
1307 	if ((ctx->ifc_flags & IFC_DMAR) == 0)
1308 		return (0);
1309 
1310 	if (!(txq->ift_sds.ifsd_map =
1311 	    (bus_dmamap_t *) malloc(sizeof(bus_dmamap_t) * sctx->isc_ntxd, M_IFLIB, M_NOWAIT | M_ZERO))) {
1312 		device_printf(dev, "Unable to allocate tx_buffer map memory\n");
1313 		err = ENOMEM;
1314 		goto fail;
1315 	}
1316 
1317 	for (int i = 0; i < sctx->isc_ntxd; i++) {
1318 		err = bus_dmamap_create(txq->ift_desc_tag, 0, &txq->ift_sds.ifsd_map[i]);
1319 		if (err != 0) {
1320 			device_printf(dev, "Unable to create TX DMA map\n");
1321 			goto fail;
1322 		}
1323 	}
1324 #endif
1325 	return (0);
1326 fail:
1327 	/* We free all, it handles case where we are in the middle */
1328 	iflib_tx_structures_free(ctx);
1329 	return (err);
1330 }
1331 
1332 static void
1333 iflib_txsd_destroy(if_ctx_t ctx, iflib_txq_t txq, int i)
1334 {
1335 	bus_dmamap_t map;
1336 
1337 	map = NULL;
1338 	if (txq->ift_sds.ifsd_map != NULL)
1339 		map = txq->ift_sds.ifsd_map[i];
1340 	if (map != NULL) {
1341 		bus_dmamap_unload(txq->ift_desc_tag, map);
1342 		bus_dmamap_destroy(txq->ift_desc_tag, map);
1343 		txq->ift_sds.ifsd_map[i] = NULL;
1344 	}
1345 }
1346 
1347 static void
1348 iflib_txq_destroy(iflib_txq_t txq)
1349 {
1350 	if_ctx_t ctx = txq->ift_ctx;
1351 	if_shared_ctx_t sctx = ctx->ifc_sctx;
1352 
1353 	for (int i = 0; i < sctx->isc_ntxd; i++)
1354 		iflib_txsd_destroy(ctx, txq, i);
1355 	if (txq->ift_sds.ifsd_map != NULL) {
1356 		free(txq->ift_sds.ifsd_map, M_IFLIB);
1357 		txq->ift_sds.ifsd_map = NULL;
1358 	}
1359 	if (txq->ift_sds.ifsd_m != NULL) {
1360 		free(txq->ift_sds.ifsd_m, M_IFLIB);
1361 		txq->ift_sds.ifsd_m = NULL;
1362 	}
1363 	if (txq->ift_sds.ifsd_flags != NULL) {
1364 		free(txq->ift_sds.ifsd_flags, M_IFLIB);
1365 		txq->ift_sds.ifsd_flags = NULL;
1366 	}
1367 	if (txq->ift_desc_tag != NULL) {
1368 		bus_dma_tag_destroy(txq->ift_desc_tag);
1369 		txq->ift_desc_tag = NULL;
1370 	}
1371 	if (txq->ift_tso_desc_tag != NULL) {
1372 		bus_dma_tag_destroy(txq->ift_tso_desc_tag);
1373 		txq->ift_tso_desc_tag = NULL;
1374 	}
1375 }
1376 
1377 static void
1378 iflib_txsd_free(if_ctx_t ctx, iflib_txq_t txq, int i)
1379 {
1380 	struct mbuf **mp;
1381 
1382 	mp = &txq->ift_sds.ifsd_m[i];
1383 	if (*mp == NULL)
1384 		return;
1385 
1386 	if (txq->ift_sds.ifsd_map != NULL) {
1387 		bus_dmamap_sync(txq->ift_desc_tag,
1388 				txq->ift_sds.ifsd_map[i],
1389 				BUS_DMASYNC_POSTWRITE);
1390 		bus_dmamap_unload(txq->ift_desc_tag,
1391 				  txq->ift_sds.ifsd_map[i]);
1392 	}
1393 	m_freem(*mp);
1394 	DBG_COUNTER_INC(tx_frees);
1395 	*mp = NULL;
1396 }
1397 
1398 static int
1399 iflib_txq_setup(iflib_txq_t txq)
1400 {
1401 	if_ctx_t ctx = txq->ift_ctx;
1402 	if_shared_ctx_t sctx = ctx->ifc_sctx;
1403 	iflib_dma_info_t di;
1404 	int i;
1405 
1406     /* Set number of descriptors available */
1407 	txq->ift_qstatus = IFLIB_QUEUE_IDLE;
1408 
1409 	/* Reset indices */
1410 	txq->ift_cidx_processed = txq->ift_pidx = txq->ift_cidx = txq->ift_npending = 0;
1411 	txq->ift_size = sctx->isc_ntxd;
1412 
1413 	for (i = 0, di = txq->ift_ifdi; i < ctx->ifc_nhwtxqs; i++, di++)
1414 		bzero((void *)di->idi_vaddr, di->idi_size);
1415 
1416 	IFDI_TXQ_SETUP(ctx, txq->ift_id);
1417 	for (i = 0, di = txq->ift_ifdi; i < ctx->ifc_nhwtxqs; i++, di++)
1418 		bus_dmamap_sync(di->idi_tag, di->idi_map,
1419 						BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1420 	return (0);
1421 }
1422 
1423 /*********************************************************************
1424  *
1425  *  Allocate memory for rx_buffer structures. Since we use one
1426  *  rx_buffer per received packet, the maximum number of rx_buffer's
1427  *  that we'll need is equal to the number of receive descriptors
1428  *  that we've allocated.
1429  *
1430  **********************************************************************/
1431 static int
1432 iflib_rxsd_alloc(iflib_rxq_t rxq)
1433 {
1434 	if_ctx_t ctx = rxq->ifr_ctx;
1435 	if_shared_ctx_t sctx = ctx->ifc_sctx;
1436 	device_t dev = ctx->ifc_dev;
1437 	iflib_fl_t fl;
1438 	iflib_rxsd_t	rxsd;
1439 	int			err;
1440 
1441 	MPASS(sctx->isc_nrxd > 0);
1442 
1443 	fl = rxq->ifr_fl;
1444 	for (int i = 0; i <  rxq->ifr_nfl; i++, fl++) {
1445 		fl->ifl_sds = malloc(sizeof(struct iflib_sw_rx_desc) *
1446 							 sctx->isc_nrxd, M_IFLIB, M_WAITOK | M_ZERO);
1447 		if (fl->ifl_sds == NULL) {
1448 			device_printf(dev, "Unable to allocate rx sw desc memory\n");
1449 			return (ENOMEM);
1450 		}
1451 		fl->ifl_size = sctx->isc_nrxd; /* this isn't necessarily the same */
1452 		err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
1453 					 1, 0,			/* alignment, bounds */
1454 					 BUS_SPACE_MAXADDR,	/* lowaddr */
1455 					 BUS_SPACE_MAXADDR,	/* highaddr */
1456 					 NULL, NULL,		/* filter, filterarg */
1457 					 sctx->isc_rx_maxsize,	/* maxsize */
1458 					 sctx->isc_rx_nsegments,	/* nsegments */
1459 					 sctx->isc_rx_maxsegsize,	/* maxsegsize */
1460 					 0,			/* flags */
1461 					 NULL,			/* lockfunc */
1462 					 NULL,			/* lockarg */
1463 					 &fl->ifl_desc_tag);
1464 		if (err) {
1465 			device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
1466 				__func__, err);
1467 			goto fail;
1468 		}
1469 
1470 		rxsd = fl->ifl_sds;
1471 		for (int i = 0; i < sctx->isc_nrxd; i++, rxsd++) {
1472 			err = bus_dmamap_create(fl->ifl_desc_tag, 0, &rxsd->ifsd_map);
1473 			if (err) {
1474 				device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
1475 					__func__, err);
1476 				goto fail;
1477 			}
1478 		}
1479 	}
1480 	return (0);
1481 
1482 fail:
1483 	iflib_rx_structures_free(ctx);
1484 	return (err);
1485 }
1486 
1487 
1488 /*
1489  * Internal service routines
1490  */
1491 
1492 struct rxq_refill_cb_arg {
1493 	int               error;
1494 	bus_dma_segment_t seg;
1495 	int               nseg;
1496 };
1497 
1498 static void
1499 _rxq_refill_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
1500 {
1501 	struct rxq_refill_cb_arg *cb_arg = arg;
1502 
1503 	cb_arg->error = error;
1504 	cb_arg->seg = segs[0];
1505 	cb_arg->nseg = nseg;
1506 }
1507 
1508 
1509 #ifdef ACPI_DMAR
1510 #define IS_DMAR(ctx) (ctx->ifc_flags & IFC_DMAR)
1511 #else
1512 #define IS_DMAR(ctx) (0)
1513 #endif
1514 
1515 /**
1516  *	rxq_refill - refill an rxq  free-buffer list
1517  *	@ctx: the iflib context
1518  *	@rxq: the free-list to refill
1519  *	@n: the number of new buffers to allocate
1520  *
1521  *	(Re)populate an rxq free-buffer list with up to @n new packet buffers.
1522  *	The caller must assure that @n does not exceed the queue's capacity.
1523  */
1524 static void
1525 _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count)
1526 {
1527 	struct mbuf *m;
1528 	int pidx = fl->ifl_pidx;
1529 	iflib_rxsd_t rxsd = &fl->ifl_sds[pidx];
1530 	caddr_t cl;
1531 	int n, i = 0;
1532 	uint64_t bus_addr;
1533 	int err;
1534 
1535 	n  = count;
1536 	MPASS(n > 0);
1537 	MPASS(fl->ifl_credits + n <= fl->ifl_size);
1538 
1539 	if (pidx < fl->ifl_cidx)
1540 		MPASS(pidx + n <= fl->ifl_cidx);
1541 	if (pidx == fl->ifl_cidx && (fl->ifl_credits < fl->ifl_size))
1542 		MPASS(fl->ifl_gen == 0);
1543 	if (pidx > fl->ifl_cidx)
1544 		MPASS(n <= fl->ifl_size - pidx + fl->ifl_cidx);
1545 
1546 	DBG_COUNTER_INC(fl_refills);
1547 	if (n > 8)
1548 		DBG_COUNTER_INC(fl_refills_large);
1549 
1550 	while (n--) {
1551 		/*
1552 		 * We allocate an uninitialized mbuf + cluster, mbuf is
1553 		 * initialized after rx.
1554 		 *
1555 		 * If the cluster is still set then we know a minimum sized packet was received
1556 		 */
1557 		if ((cl = rxsd->ifsd_cl) == NULL) {
1558 			if ((cl = rxsd->ifsd_cl = m_cljget(NULL, M_NOWAIT, fl->ifl_buf_size)) == NULL)
1559 				break;
1560 #if MEMORY_LOGGING
1561 			fl->ifl_cl_enqueued++;
1562 #endif
1563 		}
1564 		if ((m = m_gethdr(M_NOWAIT, MT_NOINIT)) == NULL) {
1565 			break;
1566 		}
1567 #if MEMORY_LOGGING
1568 		fl->ifl_m_enqueued++;
1569 #endif
1570 
1571 		DBG_COUNTER_INC(rx_allocs);
1572 #ifdef notyet
1573 		if ((rxsd->ifsd_flags & RX_SW_DESC_MAP_CREATED) == 0) {
1574 			int err;
1575 
1576 			if ((err = bus_dmamap_create(fl->ifl_ifdi->idi_tag, 0, &rxsd->ifsd_map))) {
1577 				log(LOG_WARNING, "bus_dmamap_create failed %d\n", err);
1578 				uma_zfree(fl->ifl_zone, cl);
1579 				n = 0;
1580 				goto done;
1581 			}
1582 			rxsd->ifsd_flags |= RX_SW_DESC_MAP_CREATED;
1583 		}
1584 #endif
1585 #if defined(__i386__) || defined(__amd64__)
1586 		if (!IS_DMAR(ctx)) {
1587 			bus_addr = pmap_kextract((vm_offset_t)cl);
1588 		} else
1589 #endif
1590 		{
1591 			struct rxq_refill_cb_arg cb_arg;
1592 			iflib_rxq_t q;
1593 
1594 			cb_arg.error = 0;
1595 			q = fl->ifl_rxq;
1596 			err = bus_dmamap_load(fl->ifl_desc_tag, rxsd->ifsd_map,
1597 		         cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg, 0);
1598 
1599 			if (err != 0 || cb_arg.error) {
1600 				/*
1601 				 * !zone_pack ?
1602 				 */
1603 				if (fl->ifl_zone == zone_pack)
1604 					uma_zfree(fl->ifl_zone, cl);
1605 				m_free(m);
1606 				n = 0;
1607 				goto done;
1608 			}
1609 			bus_addr = cb_arg.seg.ds_addr;
1610 		}
1611 		rxsd->ifsd_flags |= RX_SW_DESC_INUSE;
1612 
1613 		MPASS(rxsd->ifsd_m == NULL);
1614 		rxsd->ifsd_cl = cl;
1615 		rxsd->ifsd_m = m;
1616 		fl->ifl_bus_addrs[i] = bus_addr;
1617 		fl->ifl_vm_addrs[i] = cl;
1618 		rxsd++;
1619 		fl->ifl_credits++;
1620 		i++;
1621 		MPASS(fl->ifl_credits <= fl->ifl_size);
1622 		if (++fl->ifl_pidx == fl->ifl_size) {
1623 			fl->ifl_pidx = 0;
1624 			fl->ifl_gen = 1;
1625 			rxsd = fl->ifl_sds;
1626 		}
1627 		if (n == 0 || i == IFLIB_MAX_RX_REFRESH) {
1628 			ctx->isc_rxd_refill(ctx->ifc_softc, fl->ifl_rxq->ifr_id, fl->ifl_id, pidx,
1629 								 fl->ifl_bus_addrs, fl->ifl_vm_addrs, i);
1630 			i = 0;
1631 			pidx = fl->ifl_pidx;
1632 		}
1633 	}
1634 done:
1635 	DBG_COUNTER_INC(rxd_flush);
1636 	if (fl->ifl_pidx == 0)
1637 		pidx = fl->ifl_size - 1;
1638 	else
1639 		pidx = fl->ifl_pidx - 1;
1640 	ctx->isc_rxd_flush(ctx->ifc_softc, fl->ifl_rxq->ifr_id, fl->ifl_id, pidx);
1641 }
1642 
1643 static __inline void
1644 __iflib_fl_refill_lt(if_ctx_t ctx, iflib_fl_t fl, int max)
1645 {
1646 	/* we avoid allowing pidx to catch up with cidx as it confuses ixl */
1647 	int32_t reclaimable = fl->ifl_size - fl->ifl_credits - 1;
1648 #ifdef INVARIANTS
1649 	int32_t delta = fl->ifl_size - get_inuse(fl->ifl_size, fl->ifl_cidx, fl->ifl_pidx, fl->ifl_gen) - 1;
1650 #endif
1651 
1652 	MPASS(fl->ifl_credits <= fl->ifl_size);
1653 	MPASS(reclaimable == delta);
1654 
1655 	if (reclaimable > 0)
1656 		_iflib_fl_refill(ctx, fl, min(max, reclaimable));
1657 }
1658 
1659 static void
1660 iflib_fl_bufs_free(iflib_fl_t fl)
1661 {
1662 	iflib_dma_info_t idi = fl->ifl_ifdi;
1663 	uint32_t i;
1664 
1665 	for (i = 0; i < fl->ifl_size; i++) {
1666 		iflib_rxsd_t d = &fl->ifl_sds[i];
1667 
1668 		if (d->ifsd_flags & RX_SW_DESC_INUSE) {
1669 			bus_dmamap_unload(fl->ifl_desc_tag, d->ifsd_map);
1670 			bus_dmamap_destroy(fl->ifl_desc_tag, d->ifsd_map);
1671 			if (d->ifsd_m != NULL) {
1672 				m_init(d->ifsd_m, M_NOWAIT, MT_DATA, 0);
1673 				uma_zfree(zone_mbuf, d->ifsd_m);
1674 			}
1675 			if (d->ifsd_cl != NULL)
1676 				uma_zfree(fl->ifl_zone, d->ifsd_cl);
1677 			d->ifsd_flags = 0;
1678 		} else {
1679 			MPASS(d->ifsd_cl == NULL);
1680 			MPASS(d->ifsd_m == NULL);
1681 		}
1682 #if MEMORY_LOGGING
1683 		fl->ifl_m_dequeued++;
1684 		fl->ifl_cl_dequeued++;
1685 #endif
1686 		d->ifsd_cl = NULL;
1687 		d->ifsd_m = NULL;
1688 	}
1689 	/*
1690 	 * Reset free list values
1691 	 */
1692 	fl->ifl_credits = fl->ifl_cidx = fl->ifl_pidx = fl->ifl_gen = 0;;
1693 	bzero(idi->idi_vaddr, idi->idi_size);
1694 }
1695 
1696 /*********************************************************************
1697  *
1698  *  Initialize a receive ring and its buffers.
1699  *
1700  **********************************************************************/
1701 static int
1702 iflib_fl_setup(iflib_fl_t fl)
1703 {
1704 	iflib_rxq_t rxq = fl->ifl_rxq;
1705 	if_ctx_t ctx = rxq->ifr_ctx;
1706 	if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
1707 
1708 	/*
1709 	** Free current RX buffer structs and their mbufs
1710 	*/
1711 	iflib_fl_bufs_free(fl);
1712 	/* Now replenish the mbufs */
1713 	MPASS(fl->ifl_credits == 0);
1714 	/*
1715 	 * XXX don't set the max_frame_size to larger
1716 	 * than the hardware can handle
1717 	 */
1718 	if (sctx->isc_max_frame_size <= 2048)
1719 		fl->ifl_buf_size = MCLBYTES;
1720 	else if (sctx->isc_max_frame_size <= 4096)
1721 		fl->ifl_buf_size = MJUMPAGESIZE;
1722 	else if (sctx->isc_max_frame_size <= 9216)
1723 		fl->ifl_buf_size = MJUM9BYTES;
1724 	else
1725 		fl->ifl_buf_size = MJUM16BYTES;
1726 	if (fl->ifl_buf_size > ctx->ifc_max_fl_buf_size)
1727 		ctx->ifc_max_fl_buf_size = fl->ifl_buf_size;
1728 	fl->ifl_cltype = m_gettype(fl->ifl_buf_size);
1729 	fl->ifl_zone = m_getzone(fl->ifl_buf_size);
1730 
1731 
1732 	/* avoid pre-allocating zillions of clusters to an idle card
1733 	 * potentially speeding up attach
1734 	 */
1735 	_iflib_fl_refill(ctx, fl, min(128, fl->ifl_size));
1736 	MPASS(min(128, fl->ifl_size) == fl->ifl_credits);
1737 	if (min(128, fl->ifl_size) != fl->ifl_credits)
1738 		return (ENOBUFS);
1739 	/*
1740 	 * handle failure
1741 	 */
1742 	MPASS(rxq != NULL);
1743 	MPASS(fl->ifl_ifdi != NULL);
1744 	bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
1745 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1746 	return (0);
1747 }
1748 
1749 /*********************************************************************
1750  *
1751  *  Free receive ring data structures
1752  *
1753  **********************************************************************/
1754 static void
1755 iflib_rx_sds_free(iflib_rxq_t rxq)
1756 {
1757 	iflib_fl_t fl;
1758 	int i;
1759 
1760 	if (rxq->ifr_fl != NULL) {
1761 		for (i = 0; i < rxq->ifr_nfl; i++) {
1762 			fl = &rxq->ifr_fl[i];
1763 			if (fl->ifl_desc_tag != NULL) {
1764 				bus_dma_tag_destroy(fl->ifl_desc_tag);
1765 				fl->ifl_desc_tag = NULL;
1766 			}
1767 		}
1768 		if (rxq->ifr_fl->ifl_sds != NULL)
1769 			free(rxq->ifr_fl->ifl_sds, M_IFLIB);
1770 
1771 		free(rxq->ifr_fl, M_IFLIB);
1772 		rxq->ifr_fl = NULL;
1773 		rxq->ifr_cq_gen = rxq->ifr_cq_cidx = rxq->ifr_cq_pidx = 0;
1774 	}
1775 }
1776 
1777 /*
1778  * MI independent logic
1779  *
1780  */
1781 static void
1782 iflib_timer(void *arg)
1783 {
1784 	iflib_txq_t txq = arg;
1785 	if_ctx_t ctx = txq->ift_ctx;
1786 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
1787 
1788 	if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
1789 		return;
1790 	/*
1791 	** Check on the state of the TX queue(s), this
1792 	** can be done without the lock because its RO
1793 	** and the HUNG state will be static if set.
1794 	*/
1795 	IFDI_TIMER(ctx, txq->ift_id);
1796 	if ((txq->ift_qstatus == IFLIB_QUEUE_HUNG) &&
1797 		(ctx->ifc_pause_frames == 0))
1798 		goto hung;
1799 
1800 	if (TXQ_AVAIL(txq) <= 2*scctx->isc_tx_nsegments ||
1801 	    ifmp_ring_is_stalled(txq->ift_br[0]))
1802 		GROUPTASK_ENQUEUE(&txq->ift_task);
1803 
1804 	ctx->ifc_pause_frames = 0;
1805 	if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)
1806 		callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, txq->ift_timer.c_cpu);
1807 	return;
1808 hung:
1809 	CTX_LOCK(ctx);
1810 	if_setdrvflagbits(ctx->ifc_ifp, 0, IFF_DRV_RUNNING);
1811 	device_printf(ctx->ifc_dev,  "TX(%d) desc avail = %d, pidx = %d\n",
1812 				  txq->ift_id, TXQ_AVAIL(txq), txq->ift_pidx);
1813 
1814 	IFDI_WATCHDOG_RESET(ctx);
1815 	ctx->ifc_watchdog_events++;
1816 	ctx->ifc_pause_frames = 0;
1817 
1818 	iflib_init_locked(ctx);
1819 	CTX_UNLOCK(ctx);
1820 }
1821 
1822 static void
1823 iflib_init_locked(if_ctx_t ctx)
1824 {
1825 	if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
1826 	if_t ifp = ctx->ifc_ifp;
1827 	iflib_fl_t fl;
1828 	iflib_txq_t txq;
1829 	iflib_rxq_t rxq;
1830 	int i, j;
1831 
1832 
1833 	if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
1834 	IFDI_INTR_DISABLE(ctx);
1835 
1836 	/* Set hardware offload abilities */
1837 	if_clearhwassist(ifp);
1838 	if (if_getcapenable(ifp) & IFCAP_TXCSUM)
1839 		if_sethwassistbits(ifp, CSUM_IP | CSUM_TCP | CSUM_UDP, 0);
1840 	if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6)
1841 		if_sethwassistbits(ifp,  (CSUM_TCP_IPV6 | CSUM_UDP_IPV6), 0);
1842 	if (if_getcapenable(ifp) & IFCAP_TSO4)
1843 		if_sethwassistbits(ifp, CSUM_IP_TSO, 0);
1844 	if (if_getcapenable(ifp) & IFCAP_TSO6)
1845 		if_sethwassistbits(ifp, CSUM_IP6_TSO, 0);
1846 
1847 	for (i = 0, txq = ctx->ifc_txqs; i < sctx->isc_ntxqsets; i++, txq++) {
1848 		CALLOUT_LOCK(txq);
1849 		callout_stop(&txq->ift_timer);
1850 		callout_stop(&txq->ift_db_check);
1851 		CALLOUT_UNLOCK(txq);
1852 		iflib_netmap_txq_init(ctx, txq);
1853 	}
1854 	for (i = 0, rxq = ctx->ifc_rxqs; i < sctx->isc_nrxqsets; i++, rxq++) {
1855 		iflib_netmap_rxq_init(ctx, rxq);
1856 	}
1857 	IFDI_INIT(ctx);
1858 	for (i = 0, rxq = ctx->ifc_rxqs; i < sctx->isc_nrxqsets; i++, rxq++) {
1859 		for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) {
1860 			if (iflib_fl_setup(fl)) {
1861 				device_printf(ctx->ifc_dev, "freelist setup failed - check cluster settings\n");
1862 				goto done;
1863 			}
1864 		}
1865 	}
1866 	done:
1867 	if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
1868 	IFDI_INTR_ENABLE(ctx);
1869 	txq = ctx->ifc_txqs;
1870 	for (i = 0; i < sctx->isc_ntxqsets; i++, txq++)
1871 		callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq,
1872 			txq->ift_timer.c_cpu);
1873 }
1874 
1875 static int
1876 iflib_media_change(if_t ifp)
1877 {
1878 	if_ctx_t ctx = if_getsoftc(ifp);
1879 	int err;
1880 
1881 	CTX_LOCK(ctx);
1882 	if ((err = IFDI_MEDIA_CHANGE(ctx)) == 0)
1883 		iflib_init_locked(ctx);
1884 	CTX_UNLOCK(ctx);
1885 	return (err);
1886 }
1887 
1888 static void
1889 iflib_media_status(if_t ifp, struct ifmediareq *ifmr)
1890 {
1891 	if_ctx_t ctx = if_getsoftc(ifp);
1892 
1893 	CTX_LOCK(ctx);
1894 	IFDI_UPDATE_ADMIN_STATUS(ctx);
1895 	IFDI_MEDIA_STATUS(ctx, ifmr);
1896 	CTX_UNLOCK(ctx);
1897 }
1898 
1899 static void
1900 iflib_stop(if_ctx_t ctx)
1901 {
1902 	iflib_txq_t txq = ctx->ifc_txqs;
1903 	iflib_rxq_t rxq = ctx->ifc_rxqs;
1904 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
1905 	if_shared_ctx_t sctx = ctx->ifc_sctx;
1906 	iflib_dma_info_t di;
1907 	iflib_fl_t fl;
1908 	int i, j;
1909 
1910 	/* Tell the stack that the interface is no longer active */
1911 	if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
1912 
1913 	IFDI_INTR_DISABLE(ctx);
1914 	msleep(ctx, &ctx->ifc_mtx, PUSER, "iflib_init", hz);
1915 
1916 	/* Wait for current tx queue users to exit to disarm watchdog timer. */
1917 	for (i = 0; i < scctx->isc_ntxqsets; i++, txq++) {
1918 		/* make sure all transmitters have completed before proceeding XXX */
1919 
1920 		/* clean any enqueued buffers */
1921 		iflib_txq_check_drain(txq, 0);
1922 		/* Free any existing tx buffers. */
1923 		for (j = 0; j < sctx->isc_ntxd; j++) {
1924 			iflib_txsd_free(ctx, txq, j);
1925 		}
1926 		txq->ift_processed = txq->ift_cleaned = txq->ift_cidx_processed = 0;
1927 		txq->ift_in_use = txq->ift_cidx = txq->ift_pidx = txq->ift_no_desc_avail = 0;
1928 		txq->ift_closed = txq->ift_mbuf_defrag = txq->ift_mbuf_defrag_failed = 0;
1929 		txq->ift_no_tx_dma_setup = txq->ift_txd_encap_efbig = txq->ift_map_failed = 0;
1930 		txq->ift_pullups = 0;
1931 		ifmp_ring_reset_stats(txq->ift_br[0]);
1932 		for (j = 0, di = txq->ift_ifdi; j < ctx->ifc_nhwtxqs; j++, di++)
1933 			bzero((void *)di->idi_vaddr, di->idi_size);
1934 	}
1935 	for (i = 0; i < scctx->isc_nrxqsets; i++, rxq++) {
1936 		/* make sure all transmitters have completed before proceeding XXX */
1937 
1938 		for (j = 0, di = txq->ift_ifdi; j < ctx->ifc_nhwrxqs; j++, di++)
1939 			bzero((void *)di->idi_vaddr, di->idi_size);
1940 		/* also resets the free lists pidx/cidx */
1941 		for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)
1942 			iflib_fl_bufs_free(fl);
1943 	}
1944 	IFDI_STOP(ctx);
1945 }
1946 
1947 static iflib_rxsd_t
1948 rxd_frag_to_sd(iflib_rxq_t rxq, if_rxd_frag_t irf, int *cltype, int unload)
1949 {
1950 	int flid, cidx;
1951 	iflib_rxsd_t sd;
1952 	iflib_fl_t fl;
1953 	iflib_dma_info_t di;
1954 
1955 	flid = irf->irf_flid;
1956 	cidx = irf->irf_idx;
1957 	fl = &rxq->ifr_fl[flid];
1958 	fl->ifl_credits--;
1959 #if MEMORY_LOGGING
1960 	fl->ifl_m_dequeued++;
1961 	if (cltype)
1962 		fl->ifl_cl_dequeued++;
1963 #endif
1964 	sd = &fl->ifl_sds[cidx];
1965 	di = fl->ifl_ifdi;
1966 	bus_dmamap_sync(di->idi_tag, di->idi_map,
1967 			BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1968 
1969 	/* not valid assert if bxe really does SGE from non-contiguous elements */
1970 	MPASS(fl->ifl_cidx == cidx);
1971 	if (unload)
1972 		bus_dmamap_unload(fl->ifl_desc_tag, sd->ifsd_map);
1973 
1974 	if (__predict_false(++fl->ifl_cidx == fl->ifl_size)) {
1975 		fl->ifl_cidx = 0;
1976 		fl->ifl_gen = 0;
1977 	}
1978 	/* YES ick */
1979 	if (cltype)
1980 		*cltype = fl->ifl_cltype;
1981 	return (sd);
1982 }
1983 
1984 static struct mbuf *
1985 assemble_segments(iflib_rxq_t rxq, if_rxd_info_t ri)
1986 {
1987 	int i, padlen , flags, cltype;
1988 	struct mbuf *m, *mh, *mt;
1989 	iflib_rxsd_t sd;
1990 	caddr_t cl;
1991 
1992 	i = 0;
1993 	do {
1994 		sd = rxd_frag_to_sd(rxq, &ri->iri_frags[i], &cltype, TRUE);
1995 
1996 		MPASS(sd->ifsd_cl != NULL);
1997 		MPASS(sd->ifsd_m != NULL);
1998 		m = sd->ifsd_m;
1999 		if (i == 0) {
2000 			flags = M_PKTHDR|M_EXT;
2001 			mh = mt = m;
2002 			padlen = ri->iri_pad;
2003 		} else {
2004 			flags = M_EXT;
2005 			mt->m_next = m;
2006 			mt = m;
2007 			/* assuming padding is only on the first fragment */
2008 			padlen = 0;
2009 		}
2010 		sd->ifsd_m = NULL;
2011 		cl = sd->ifsd_cl;
2012 		sd->ifsd_cl = NULL;
2013 
2014 		/* Can these two be made one ? */
2015 		m_init(m, M_NOWAIT, MT_DATA, flags);
2016 		m_cljset(m, cl, cltype);
2017 		/*
2018 		 * These must follow m_init and m_cljset
2019 		 */
2020 		m->m_data += padlen;
2021 		ri->iri_len -= padlen;
2022 		m->m_len = ri->iri_len;
2023 	} while (++i < ri->iri_nfrags);
2024 
2025 	return (mh);
2026 }
2027 
2028 
2029 
2030 /*
2031  * Process one software descriptor
2032  */
2033 static struct mbuf *
2034 iflib_rxd_pkt_get(iflib_rxq_t rxq, if_rxd_info_t ri)
2035 {
2036 	struct mbuf *m;
2037 	iflib_rxsd_t sd;
2038 
2039 	/* should I merge this back in now that the two paths are basically duplicated? */
2040 	if (ri->iri_len <= IFLIB_RX_COPY_THRESH) {
2041 		sd = rxd_frag_to_sd(rxq, &ri->iri_frags[0], NULL, FALSE);
2042 		m = sd->ifsd_m;
2043 		sd->ifsd_m = NULL;
2044 		m_init(m, M_NOWAIT, MT_DATA, M_PKTHDR);
2045 		memcpy(m->m_data, sd->ifsd_cl, ri->iri_len);
2046 		m->m_len = ri->iri_len;
2047        } else {
2048 		m = assemble_segments(rxq, ri);
2049 	}
2050 	m->m_pkthdr.len = ri->iri_len;
2051 	m->m_pkthdr.rcvif = ri->iri_ifp;
2052 	m->m_flags |= ri->iri_flags;
2053 	m->m_pkthdr.ether_vtag = ri->iri_vtag;
2054 	m->m_pkthdr.flowid = ri->iri_flowid;
2055 	M_HASHTYPE_SET(m, ri->iri_rsstype);
2056 	m->m_pkthdr.csum_flags = ri->iri_csum_flags;
2057 	m->m_pkthdr.csum_data = ri->iri_csum_data;
2058 	return (m);
2059 }
2060 
2061 static bool
2062 iflib_rxeof(iflib_rxq_t rxq, int budget)
2063 {
2064 	if_ctx_t ctx = rxq->ifr_ctx;
2065 	if_shared_ctx_t sctx = ctx->ifc_sctx;
2066 	int avail, i;
2067 	uint16_t *cidxp;
2068 	struct if_rxd_info ri;
2069 	int err, budget_left, rx_bytes, rx_pkts;
2070 	iflib_fl_t fl;
2071 	struct ifnet *ifp;
2072 	struct lro_entry *queued;
2073 	int lro_enabled;
2074 	/*
2075 	 * XXX early demux data packets so that if_input processing only handles
2076 	 * acks in interrupt context
2077 	 */
2078 	struct mbuf *m, *mh, *mt;
2079 
2080 	if (netmap_rx_irq(ctx->ifc_ifp, rxq->ifr_id, &budget)) {
2081 		return (FALSE);
2082 	}
2083 
2084 	mh = mt = NULL;
2085 	MPASS(budget > 0);
2086 	rx_pkts	= rx_bytes = 0;
2087 	if (sctx->isc_flags & IFLIB_HAS_CQ)
2088 		cidxp = &rxq->ifr_cq_cidx;
2089 	else
2090 		cidxp = &rxq->ifr_fl[0].ifl_cidx;
2091 	if ((avail = iflib_rxd_avail(ctx, rxq, *cidxp)) == 0) {
2092 		for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++)
2093 			__iflib_fl_refill_lt(ctx, fl, budget + 8);
2094 		DBG_COUNTER_INC(rx_unavail);
2095 		return (false);
2096 	}
2097 
2098 	for (budget_left = budget; (budget_left > 0) && (avail > 0); budget_left--, avail--) {
2099 		if (__predict_false(!CTX_ACTIVE(ctx))) {
2100 			DBG_COUNTER_INC(rx_ctx_inactive);
2101 			break;
2102 		}
2103 		/*
2104 		 * Reset client set fields to their default values
2105 		 */
2106 		bzero(&ri, sizeof(ri));
2107 		ri.iri_qsidx = rxq->ifr_id;
2108 		ri.iri_cidx = *cidxp;
2109 		ri.iri_ifp = ctx->ifc_ifp;
2110 		ri.iri_frags = rxq->ifr_frags;
2111 		err = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri);
2112 
2113 		/* in lieu of handling correctly - make sure it isn't being unhandled */
2114 		MPASS(err == 0);
2115 		if (sctx->isc_flags & IFLIB_HAS_CQ) {
2116 			/* we know we consumed _one_ CQ entry */
2117 			if (++rxq->ifr_cq_cidx == sctx->isc_nrxd) {
2118 				rxq->ifr_cq_cidx = 0;
2119 				rxq->ifr_cq_gen = 0;
2120 			}
2121 			/* was this only a completion queue message? */
2122 			if (__predict_false(ri.iri_nfrags == 0))
2123 				continue;
2124 		}
2125 		MPASS(ri.iri_nfrags != 0);
2126 		MPASS(ri.iri_len != 0);
2127 
2128 		/* will advance the cidx on the corresponding free lists */
2129 		m = iflib_rxd_pkt_get(rxq, &ri);
2130 		if (avail == 0 && budget_left)
2131 			avail = iflib_rxd_avail(ctx, rxq, *cidxp);
2132 
2133 		if (__predict_false(m == NULL)) {
2134 			DBG_COUNTER_INC(rx_mbuf_null);
2135 			continue;
2136 		}
2137 		/* imm_pkt: -- cxgb */
2138 		if (mh == NULL)
2139 			mh = mt = m;
2140 		else {
2141 			mt->m_nextpkt = m;
2142 			mt = m;
2143 		}
2144 	}
2145 	/* make sure that we can refill faster than drain */
2146 	for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++)
2147 		__iflib_fl_refill_lt(ctx, fl, budget + 8);
2148 
2149 	ifp = ctx->ifc_ifp;
2150 	lro_enabled = (if_getcapenable(ifp) & IFCAP_LRO);
2151 
2152 	while (mh != NULL) {
2153 		m = mh;
2154 		mh = mh->m_nextpkt;
2155 		m->m_nextpkt = NULL;
2156 		rx_bytes += m->m_pkthdr.len;
2157 		rx_pkts++;
2158 #if defined(INET6) || defined(INET)
2159 		if (lro_enabled && tcp_lro_rx(&rxq->ifr_lc, m, 0) == 0)
2160 			continue;
2161 #endif
2162 		DBG_COUNTER_INC(rx_if_input);
2163 		ifp->if_input(ifp, m);
2164 	}
2165 	if_inc_counter(ifp, IFCOUNTER_IBYTES, rx_bytes);
2166 	if_inc_counter(ifp, IFCOUNTER_IPACKETS, rx_pkts);
2167 
2168 	/*
2169 	 * Flush any outstanding LRO work
2170 	 */
2171 	while ((queued = LIST_FIRST(&rxq->ifr_lc.lro_active)) != NULL) {
2172 		LIST_REMOVE(queued, next);
2173 #if defined(INET6) || defined(INET)
2174 		tcp_lro_flush(&rxq->ifr_lc, queued);
2175 #endif
2176 	}
2177 	return (iflib_rxd_avail(ctx, rxq, *cidxp));
2178 }
2179 
2180 #define M_CSUM_FLAGS(m) ((m)->m_pkthdr.csum_flags)
2181 #define M_HAS_VLANTAG(m) (m->m_flags & M_VLANTAG)
2182 #define TXQ_MAX_DB_DEFERRED(ctx) (ctx->ifc_sctx->isc_ntxd >> 5)
2183 #define TXQ_MAX_DB_CONSUMED(ctx) (ctx->ifc_sctx->isc_ntxd >> 4)
2184 
2185 static __inline void
2186 iflib_txd_db_check(if_ctx_t ctx, iflib_txq_t txq, int ring)
2187 {
2188 	uint32_t dbval;
2189 
2190 	if (ring || txq->ift_db_pending >= TXQ_MAX_DB_DEFERRED(ctx)) {
2191 
2192 		/* the lock will only ever be contended in the !min_latency case */
2193 		if (!TXDB_TRYLOCK(txq))
2194 			return;
2195 		dbval = txq->ift_npending ? txq->ift_npending : txq->ift_pidx;
2196 		ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, dbval);
2197 		txq->ift_db_pending = txq->ift_npending = 0;
2198 		TXDB_UNLOCK(txq);
2199 	}
2200 }
2201 
2202 static void
2203 iflib_txd_deferred_db_check(void * arg)
2204 {
2205 	iflib_txq_t txq = arg;
2206 
2207 	/* simple non-zero boolean so use bitwise OR */
2208 	if ((txq->ift_db_pending | txq->ift_npending) &&
2209 	    txq->ift_db_pending >= txq->ift_db_pending_queued)
2210 		iflib_txd_db_check(txq->ift_ctx, txq, TRUE);
2211 	txq->ift_db_pending_queued = 0;
2212 	if (ifmp_ring_is_stalled(txq->ift_br[0]))
2213 		iflib_txq_check_drain(txq, 4);
2214 }
2215 
2216 #ifdef PKT_DEBUG
2217 static void
2218 print_pkt(if_pkt_info_t pi)
2219 {
2220 	printf("pi len:  %d qsidx: %d nsegs: %d ndescs: %d flags: %x pidx: %d\n",
2221 	       pi->ipi_len, pi->ipi_qsidx, pi->ipi_nsegs, pi->ipi_ndescs, pi->ipi_flags, pi->ipi_pidx);
2222 	printf("pi new_pidx: %d csum_flags: %lx tso_segsz: %d mflags: %x vtag: %d\n",
2223 	       pi->ipi_new_pidx, pi->ipi_csum_flags, pi->ipi_tso_segsz, pi->ipi_mflags, pi->ipi_vtag);
2224 	printf("pi etype: %d ehdrlen: %d ip_hlen: %d ipproto: %d\n",
2225 	       pi->ipi_etype, pi->ipi_ehdrlen, pi->ipi_ip_hlen, pi->ipi_ipproto);
2226 }
2227 #endif
2228 
2229 #define IS_TSO4(pi) ((pi)->ipi_csum_flags & CSUM_IP_TSO)
2230 #define IS_TSO6(pi) ((pi)->ipi_csum_flags & CSUM_IP6_TSO)
2231 
2232 static int
2233 iflib_parse_header(iflib_txq_t txq, if_pkt_info_t pi, struct mbuf **mp)
2234 {
2235 	struct ether_vlan_header *eh;
2236 	struct mbuf *m;
2237 
2238 	m = *mp;
2239 	/*
2240 	 * Determine where frame payload starts.
2241 	 * Jump over vlan headers if already present,
2242 	 * helpful for QinQ too.
2243 	 */
2244 	if (__predict_false(m->m_len < sizeof(*eh))) {
2245 		txq->ift_pullups++;
2246 		if (__predict_false((m = m_pullup(m, sizeof(*eh))) == NULL))
2247 			return (ENOMEM);
2248 	}
2249 	eh = mtod(m, struct ether_vlan_header *);
2250 	if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
2251 		pi->ipi_etype = ntohs(eh->evl_proto);
2252 		pi->ipi_ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
2253 	} else {
2254 		pi->ipi_etype = ntohs(eh->evl_encap_proto);
2255 		pi->ipi_ehdrlen = ETHER_HDR_LEN;
2256 	}
2257 
2258 	switch (pi->ipi_etype) {
2259 #ifdef INET
2260 	case ETHERTYPE_IP:
2261 	{
2262 		struct ip *ip = NULL;
2263 		struct tcphdr *th = NULL;
2264 		struct mbuf *n;
2265 		int minthlen;
2266 
2267 		minthlen = min(m->m_pkthdr.len, pi->ipi_ehdrlen + sizeof(*ip) + sizeof(*th));
2268 		if (__predict_false(m->m_len < minthlen)) {
2269 			/*
2270 			 * if this code bloat is causing too much of a hit
2271 			 * move it to a separate function and mark it noinline
2272 			 */
2273 			if (m->m_len == pi->ipi_ehdrlen) {
2274 				n = m->m_next;
2275 				MPASS(n);
2276 				if (n->m_len >= sizeof(*ip))  {
2277 					ip = (struct ip *)n->m_data;
2278 					if (n->m_len >= (ip->ip_hl << 2) + sizeof(*th))
2279 						th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
2280 				} else {
2281 					txq->ift_pullups++;
2282 					if (__predict_false((m = m_pullup(m, minthlen)) == NULL))
2283 						return (ENOMEM);
2284 					ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen);
2285 				}
2286 			} else {
2287 				txq->ift_pullups++;
2288 				if (__predict_false((m = m_pullup(m, minthlen)) == NULL))
2289 					return (ENOMEM);
2290 				ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen);
2291 				if (m->m_len >= (ip->ip_hl << 2) + sizeof(*th))
2292 					th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
2293 			}
2294 		} else {
2295 			ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen);
2296 			if (m->m_len >= (ip->ip_hl << 2) + sizeof(*th))
2297 				th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
2298 		}
2299 		pi->ipi_ip_hlen = ip->ip_hl << 2;
2300 		pi->ipi_ipproto = ip->ip_p;
2301 		pi->ipi_flags |= IPI_TX_IPV4;
2302 
2303 		if (pi->ipi_csum_flags & CSUM_IP)
2304                        ip->ip_sum = 0;
2305 
2306 		if (pi->ipi_ipproto == IPPROTO_TCP) {
2307 			if (__predict_false(th == NULL)) {
2308 				txq->ift_pullups++;
2309 				if (__predict_false((m = m_pullup(m, (ip->ip_hl << 2) + sizeof(*th))) == NULL))
2310 					return (ENOMEM);
2311 				th = (struct tcphdr *)((caddr_t)ip + pi->ipi_ip_hlen);
2312 			}
2313 			pi->ipi_tcp_hflags = th->th_flags;
2314 			pi->ipi_tcp_hlen = th->th_off << 2;
2315 			pi->ipi_tcp_seq = th->th_seq;
2316 		}
2317 		if (IS_TSO4(pi)) {
2318 			if (__predict_false(ip->ip_p != IPPROTO_TCP))
2319 				return (ENXIO);
2320 			th->th_sum = in_pseudo(ip->ip_src.s_addr,
2321 					       ip->ip_dst.s_addr, htons(IPPROTO_TCP));
2322 			pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz;
2323 		}
2324 		break;
2325 	}
2326 #endif
2327 #ifdef INET6
2328 	case ETHERTYPE_IPV6:
2329 	{
2330 		struct ip6_hdr *ip6 = (struct ip6_hdr *)(m->m_data + pi->ipi_ehdrlen);
2331 		struct tcphdr *th;
2332 		pi->ipi_ip_hlen = sizeof(struct ip6_hdr);
2333 
2334 		if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr))) {
2335 			if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr))) == NULL))
2336 				return (ENOMEM);
2337 		}
2338 		th = (struct tcphdr *)((caddr_t)ip6 + pi->ipi_ip_hlen);
2339 
2340 		/* XXX-BZ this will go badly in case of ext hdrs. */
2341 		pi->ipi_ipproto = ip6->ip6_nxt;
2342 		pi->ipi_flags |= IPI_TX_IPV6;
2343 
2344 		if (pi->ipi_ipproto == IPPROTO_TCP) {
2345 			if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) {
2346 				if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) == NULL))
2347 					return (ENOMEM);
2348 			}
2349 			pi->ipi_tcp_hflags = th->th_flags;
2350 			pi->ipi_tcp_hlen = th->th_off << 2;
2351 		}
2352 		if (IS_TSO6(pi)) {
2353 
2354 			if (__predict_false(ip6->ip6_nxt != IPPROTO_TCP))
2355 				return (ENXIO);
2356 			/*
2357 			 * The corresponding flag is set by the stack in the IPv4
2358 			 * TSO case, but not in IPv6 (at least in FreeBSD 10.2).
2359 			 * So, set it here because the rest of the flow requires it.
2360 			 */
2361 			pi->ipi_csum_flags |= CSUM_TCP_IPV6;
2362 			th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
2363 			pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz;
2364 		}
2365 		break;
2366 	}
2367 #endif
2368 	default:
2369 		pi->ipi_csum_flags &= ~CSUM_OFFLOAD;
2370 		pi->ipi_ip_hlen = 0;
2371 		break;
2372 	}
2373 	*mp = m;
2374 	return (0);
2375 }
2376 
2377 
2378 static  __noinline  struct mbuf *
2379 collapse_pkthdr(struct mbuf *m0)
2380 {
2381 	struct mbuf *m, *m_next, *tmp;
2382 
2383 	m = m0;
2384 	m_next = m->m_next;
2385 	while (m_next != NULL && m_next->m_len == 0) {
2386 		m = m_next;
2387 		m->m_next = NULL;
2388 		m_free(m);
2389 		m_next = m_next->m_next;
2390 	}
2391 	m = m0;
2392 	m->m_next = m_next;
2393 	if ((m_next->m_flags & M_EXT) == 0) {
2394 		m = m_defrag(m, M_NOWAIT);
2395 	} else {
2396 		tmp = m_next->m_next;
2397 		memcpy(m_next, m, MPKTHSIZE);
2398 		m = m_next;
2399 		m->m_next = tmp;
2400 	}
2401 	return (m);
2402 }
2403 
2404 /*
2405  * If dodgy hardware rejects the scatter gather chain we've handed it
2406  * we'll need to rebuild the mbuf chain before we can call m_defrag
2407  */
2408 static __noinline struct mbuf *
2409 iflib_rebuild_mbuf(iflib_txq_t txq)
2410 {
2411 
2412 	int ntxd, mhlen, len, i, pidx;
2413 	struct mbuf *m, *mh, **ifsd_m;
2414 	if_shared_ctx_t		sctx;
2415 
2416 	pidx = txq->ift_pidx;
2417 	ifsd_m = txq->ift_sds.ifsd_m;
2418 	sctx = txq->ift_ctx->ifc_sctx;
2419 	ntxd = sctx->isc_ntxd;
2420 	mh = m = ifsd_m[pidx];
2421 	ifsd_m[pidx] = NULL;
2422 #if MEMORY_LOGGING
2423 	txq->ift_dequeued++;
2424 #endif
2425 	len = m->m_len;
2426 	mhlen = m->m_pkthdr.len;
2427 	i = 1;
2428 
2429 	while (len < mhlen && (m->m_next == NULL)) {
2430 		m->m_next = ifsd_m[(pidx + i) & (ntxd-1)];
2431 		ifsd_m[(pidx + i) & (ntxd -1)] = NULL;
2432 #if MEMORY_LOGGING
2433 		txq->ift_dequeued++;
2434 #endif
2435 		m = m->m_next;
2436 		len += m->m_len;
2437 		i++;
2438 	}
2439 	return (mh);
2440 }
2441 
2442 static int
2443 iflib_busdma_load_mbuf_sg(iflib_txq_t txq, bus_dma_tag_t tag, bus_dmamap_t map,
2444 			  struct mbuf **m0, bus_dma_segment_t *segs, int *nsegs,
2445 			  int max_segs, int flags)
2446 {
2447 	if_ctx_t ctx;
2448 	if_shared_ctx_t		sctx;
2449 	int i, next, pidx, mask, err, maxsegsz, ntxd, count;
2450 	struct mbuf *m, *tmp, **ifsd_m, **mp;
2451 
2452 	m = *m0;
2453 
2454 	/*
2455 	 * Please don't ever do this
2456 	 */
2457 	if (__predict_false(m->m_len == 0))
2458 		*m0 = m = collapse_pkthdr(m);
2459 
2460 	ctx = txq->ift_ctx;
2461 	sctx = ctx->ifc_sctx;
2462 	ifsd_m = txq->ift_sds.ifsd_m;
2463 	ntxd = sctx->isc_ntxd;
2464 	pidx = txq->ift_pidx;
2465 	if (map != NULL) {
2466 		uint8_t *ifsd_flags = txq->ift_sds.ifsd_flags;
2467 
2468 		err = bus_dmamap_load_mbuf_sg(tag, map,
2469 					      *m0, segs, nsegs, BUS_DMA_NOWAIT);
2470 		if (err)
2471 			return (err);
2472 		ifsd_flags[pidx] |= TX_SW_DESC_MAPPED;
2473 		i = 0;
2474 		next = pidx;
2475 		mask = (sctx->isc_ntxd-1);
2476 		m = *m0;
2477 		do {
2478 			mp = &ifsd_m[next];
2479 			*mp = m;
2480 			m = m->m_next;
2481 			(*mp)->m_next = NULL;
2482 			if (__predict_false((*mp)->m_len == 0)) {
2483 				m_free(*mp);
2484 				*mp = NULL;
2485 			} else
2486 				next = (pidx + i) & (ntxd-1);
2487 		} while (m != NULL);
2488 	} else {
2489 		int buflen, sgsize, max_sgsize;
2490 		vm_offset_t vaddr;
2491 		vm_paddr_t curaddr;
2492 
2493 		count = i = 0;
2494 		maxsegsz = sctx->isc_tx_maxsize;
2495 		m = *m0;
2496 		do {
2497 			if (__predict_false(m->m_len <= 0)) {
2498 				tmp = m;
2499 				m = m->m_next;
2500 				tmp->m_next = NULL;
2501 				m_free(tmp);
2502 				continue;
2503 			}
2504 			buflen = m->m_len;
2505 			vaddr = (vm_offset_t)m->m_data;
2506 			/*
2507 			 * see if we can't be smarter about physically
2508 			 * contiguous mappings
2509 			 */
2510 			next = (pidx + count) & (ntxd-1);
2511 			MPASS(ifsd_m[next] == NULL);
2512 #if MEMORY_LOGGING
2513 			txq->ift_enqueued++;
2514 #endif
2515 			ifsd_m[next] = m;
2516 			while (buflen > 0) {
2517 				max_sgsize = MIN(buflen, maxsegsz);
2518 				curaddr = pmap_kextract(vaddr);
2519 				sgsize = PAGE_SIZE - (curaddr & PAGE_MASK);
2520 				sgsize = MIN(sgsize, max_sgsize);
2521 				segs[i].ds_addr = curaddr;
2522 				segs[i].ds_len = sgsize;
2523 				vaddr += sgsize;
2524 				buflen -= sgsize;
2525 				i++;
2526 				if (i >= max_segs)
2527 					goto err;
2528 			}
2529 			count++;
2530 			tmp = m;
2531 			m = m->m_next;
2532 			tmp->m_next = NULL;
2533 		} while (m != NULL);
2534 		*nsegs = i;
2535 	}
2536 	return (0);
2537 err:
2538 	*m0 = iflib_rebuild_mbuf(txq);
2539 	return (EFBIG);
2540 }
2541 
2542 static int
2543 iflib_encap(iflib_txq_t txq, struct mbuf **m_headp)
2544 {
2545 	if_ctx_t		ctx;
2546 	if_shared_ctx_t		sctx;
2547 	if_softc_ctx_t		scctx;
2548 	bus_dma_segment_t	*segs;
2549 	struct mbuf		*m_head;
2550 	bus_dmamap_t		map;
2551 	struct if_pkt_info	pi;
2552 	int remap = 0;
2553 	int err, nsegs, ndesc, max_segs, pidx, cidx, next, ntxd;
2554 	bus_dma_tag_t desc_tag;
2555 
2556 	segs = txq->ift_segs;
2557 	ctx = txq->ift_ctx;
2558 	sctx = ctx->ifc_sctx;
2559 	scctx = &ctx->ifc_softc_ctx;
2560 	segs = txq->ift_segs;
2561 	ntxd = sctx->isc_ntxd;
2562 	m_head = *m_headp;
2563 	map = NULL;
2564 
2565 	/*
2566 	 * If we're doing TSO the next descriptor to clean may be quite far ahead
2567 	 */
2568 	cidx = txq->ift_cidx;
2569 	pidx = txq->ift_pidx;
2570 	next = (cidx + CACHE_PTR_INCREMENT) & (ntxd-1);
2571 
2572 	/* prefetch the next cache line of mbuf pointers and flags */
2573 	prefetch(&txq->ift_sds.ifsd_m[next]);
2574 	if (txq->ift_sds.ifsd_map != NULL) {
2575 		prefetch(&txq->ift_sds.ifsd_map[next]);
2576 		map = txq->ift_sds.ifsd_map[pidx];
2577 		next = (cidx + CACHE_LINE_SIZE) & (ntxd-1);
2578 		prefetch(&txq->ift_sds.ifsd_flags[next]);
2579 	}
2580 
2581 
2582 	if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
2583 		desc_tag = txq->ift_tso_desc_tag;
2584 		max_segs = scctx->isc_tx_tso_segments_max;
2585 	} else {
2586 		desc_tag = txq->ift_desc_tag;
2587 		max_segs = scctx->isc_tx_nsegments;
2588 	}
2589 	m_head = *m_headp;
2590 	bzero(&pi, sizeof(pi));
2591 	pi.ipi_len = m_head->m_pkthdr.len;
2592 	pi.ipi_mflags = (m_head->m_flags & (M_VLANTAG|M_BCAST|M_MCAST));
2593 	pi.ipi_csum_flags = m_head->m_pkthdr.csum_flags;
2594 	pi.ipi_vtag = (m_head->m_flags & M_VLANTAG) ? m_head->m_pkthdr.ether_vtag : 0;
2595 	pi.ipi_pidx = pidx;
2596 	pi.ipi_qsidx = txq->ift_id;
2597 
2598 	/* deliberate bitwise OR to make one condition */
2599 	if (__predict_true((pi.ipi_csum_flags | pi.ipi_vtag))) {
2600 		if (__predict_false((err = iflib_parse_header(txq, &pi, m_headp)) != 0))
2601 			return (err);
2602 		m_head = *m_headp;
2603 	}
2604 
2605 retry:
2606 	err = iflib_busdma_load_mbuf_sg(txq, desc_tag, map, m_headp, segs, &nsegs, max_segs, BUS_DMA_NOWAIT);
2607 defrag:
2608 	if (__predict_false(err)) {
2609 		switch (err) {
2610 		case EFBIG:
2611 			/* try collapse once and defrag once */
2612 			if (remap == 0)
2613 				m_head = m_collapse(*m_headp, M_NOWAIT, max_segs);
2614 			if (remap == 1)
2615 				m_head = m_defrag(*m_headp, M_NOWAIT);
2616 			remap++;
2617 			if (__predict_false(m_head == NULL))
2618 				goto defrag_failed;
2619 			txq->ift_mbuf_defrag++;
2620 			*m_headp = m_head;
2621 			goto retry;
2622 			break;
2623 		case ENOMEM:
2624 			txq->ift_no_tx_dma_setup++;
2625 			break;
2626 		default:
2627 			txq->ift_no_tx_dma_setup++;
2628 			m_freem(*m_headp);
2629 			DBG_COUNTER_INC(tx_frees);
2630 			*m_headp = NULL;
2631 			break;
2632 		}
2633 		txq->ift_map_failed++;
2634 		DBG_COUNTER_INC(encap_load_mbuf_fail);
2635 		return (err);
2636 	}
2637 
2638 	/*
2639 	 * XXX assumes a 1 to 1 relationship between segments and
2640 	 *        descriptors - this does not hold true on all drivers, e.g.
2641 	 *        cxgb
2642 	 */
2643 	if (__predict_false(nsegs + 2 > TXQ_AVAIL(txq))) {
2644 		txq->ift_no_desc_avail++;
2645 		if (map != NULL)
2646 			bus_dmamap_unload(desc_tag, map);
2647 		DBG_COUNTER_INC(encap_txq_avail_fail);
2648 		if (txq->ift_task.gt_task.ta_pending == 0)
2649 			GROUPTASK_ENQUEUE(&txq->ift_task);
2650 		return (ENOBUFS);
2651 	}
2652 	pi.ipi_segs = segs;
2653 	pi.ipi_nsegs = nsegs;
2654 
2655 	MPASS(pidx >= 0 && pidx < sctx->isc_ntxd);
2656 #ifdef PKT_DEBUG
2657 	print_pkt(&pi);
2658 #endif
2659 	if ((err = ctx->isc_txd_encap(ctx->ifc_softc, &pi)) == 0) {
2660 		bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
2661 						BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2662 
2663 		DBG_COUNTER_INC(tx_encap);
2664 		MPASS(pi.ipi_new_pidx >= 0 && pi.ipi_new_pidx < sctx->isc_ntxd);
2665 
2666 		ndesc = pi.ipi_new_pidx - pi.ipi_pidx;
2667 		if (pi.ipi_new_pidx < pi.ipi_pidx) {
2668 			ndesc += sctx->isc_ntxd;
2669 			txq->ift_gen = 1;
2670 		}
2671 		MPASS(pi.ipi_new_pidx != pidx);
2672 		MPASS(ndesc > 0);
2673 		txq->ift_in_use += ndesc;
2674 		/*
2675 		 * We update the last software descriptor again here because there may
2676 		 * be a sentinel and/or there may be more mbufs than segments
2677 		 */
2678 		txq->ift_pidx = pi.ipi_new_pidx;
2679 		txq->ift_npending += pi.ipi_ndescs;
2680 	} else if (__predict_false(err == EFBIG && remap < 2)) {
2681 		*m_headp = m_head = iflib_rebuild_mbuf(txq);
2682 		remap = 1;
2683 		txq->ift_txd_encap_efbig++;
2684 		goto defrag;
2685 	} else
2686 		DBG_COUNTER_INC(encap_txd_encap_fail);
2687 	return (err);
2688 
2689 defrag_failed:
2690 	txq->ift_mbuf_defrag_failed++;
2691 	txq->ift_map_failed++;
2692 	m_freem(*m_headp);
2693 	DBG_COUNTER_INC(tx_frees);
2694 	*m_headp = NULL;
2695 	return (ENOMEM);
2696 }
2697 
2698 /* forward compatibility for cxgb */
2699 #define FIRST_QSET(ctx) 0
2700 
2701 #define NTXQSETS(ctx) ((ctx)->ifc_softc_ctx.isc_ntxqsets)
2702 #define NRXQSETS(ctx) ((ctx)->ifc_softc_ctx.isc_nrxqsets)
2703 #define QIDX(ctx, m) ((((m)->m_pkthdr.flowid & ctx->ifc_softc_ctx.isc_rss_table_mask) % NRXQSETS(ctx)) + FIRST_QSET(ctx))
2704 #define DESC_RECLAIMABLE(q) ((int)((q)->ift_processed - (q)->ift_cleaned - (q)->ift_ctx->ifc_softc_ctx.isc_tx_nsegments))
2705 #define RECLAIM_THRESH(ctx) ((ctx)->ifc_sctx->isc_tx_reclaim_thresh)
2706 #define MAX_TX_DESC(ctx) ((ctx)->ifc_softc_ctx.isc_tx_tso_segments_max)
2707 
2708 
2709 
2710 /* if there are more than TXQ_MIN_OCCUPANCY packets pending we consider deferring
2711  * doorbell writes
2712  *
2713  * ORing with 2 assures that min occupancy is never less than 2 without any conditional logic
2714  */
2715 #define TXQ_MIN_OCCUPANCY(ctx) ((ctx->ifc_sctx->isc_ntxd >> 6)| 0x2)
2716 
2717 static inline int
2718 iflib_txq_min_occupancy(iflib_txq_t txq)
2719 {
2720 	if_ctx_t ctx;
2721 
2722 	ctx = txq->ift_ctx;
2723 	return (get_inuse(txq->ift_size, txq->ift_cidx, txq->ift_pidx, txq->ift_gen) < TXQ_MIN_OCCUPANCY(ctx) + MAX_TX_DESC(ctx));
2724 }
2725 
2726 static void
2727 iflib_tx_desc_free(iflib_txq_t txq, int n)
2728 {
2729 	int hasmap;
2730 	uint32_t qsize, cidx, mask, gen;
2731 	struct mbuf *m, **ifsd_m;
2732 	uint8_t *ifsd_flags;
2733 	bus_dmamap_t *ifsd_map;
2734 
2735 	cidx = txq->ift_cidx;
2736 	gen = txq->ift_gen;
2737 	qsize = txq->ift_ctx->ifc_sctx->isc_ntxd;
2738 	mask = qsize-1;
2739 	hasmap = txq->ift_sds.ifsd_map != NULL;
2740 	ifsd_flags = txq->ift_sds.ifsd_flags;
2741 	ifsd_m = txq->ift_sds.ifsd_m;
2742 	ifsd_map = txq->ift_sds.ifsd_map;
2743 
2744 	while (n--) {
2745 		prefetch(ifsd_m[(cidx + 3) & mask]);
2746 		prefetch(ifsd_m[(cidx + 4) & mask]);
2747 
2748 		if (ifsd_m[cidx] != NULL) {
2749 			prefetch(&ifsd_m[(cidx + CACHE_PTR_INCREMENT) & mask]);
2750 			prefetch(&ifsd_flags[(cidx + CACHE_PTR_INCREMENT) & mask]);
2751 			if (hasmap && (ifsd_flags[cidx] & TX_SW_DESC_MAPPED)) {
2752 				/*
2753 				 * does it matter if it's not the TSO tag? If so we'll
2754 				 * have to add the type to flags
2755 				 */
2756 				bus_dmamap_unload(txq->ift_desc_tag, ifsd_map[cidx]);
2757 				ifsd_flags[cidx] &= ~TX_SW_DESC_MAPPED;
2758 			}
2759 			if ((m = ifsd_m[cidx]) != NULL) {
2760 				/* XXX we don't support any drivers that batch packets yet */
2761 				MPASS(m->m_nextpkt == NULL);
2762 
2763 				m_freem(m);
2764 				ifsd_m[cidx] = NULL;
2765 #if MEMORY_LOGGING
2766 				txq->ift_dequeued++;
2767 #endif
2768 				DBG_COUNTER_INC(tx_frees);
2769 			}
2770 		}
2771 		if (__predict_false(++cidx == qsize)) {
2772 			cidx = 0;
2773 			gen = 0;
2774 		}
2775 	}
2776 	txq->ift_cidx = cidx;
2777 	txq->ift_gen = gen;
2778 }
2779 
2780 static __inline int
2781 iflib_completed_tx_reclaim(iflib_txq_t txq, int thresh)
2782 {
2783 	int reclaim;
2784 	if_ctx_t ctx = txq->ift_ctx;
2785 
2786 	KASSERT(thresh >= 0, ("invalid threshold to reclaim"));
2787 	MPASS(thresh /*+ MAX_TX_DESC(txq->ift_ctx) */ < txq->ift_size);
2788 
2789 	/*
2790 	 * Need a rate-limiting check so that this isn't called every time
2791 	 */
2792 	iflib_tx_credits_update(ctx, txq);
2793 	reclaim = DESC_RECLAIMABLE(txq);
2794 
2795 	if (reclaim <= thresh /* + MAX_TX_DESC(txq->ift_ctx) */) {
2796 #ifdef INVARIANTS
2797 		if (iflib_verbose_debug) {
2798 			printf("%s processed=%ju cleaned=%ju tx_nsegments=%d reclaim=%d thresh=%d\n", __FUNCTION__,
2799 			       txq->ift_processed, txq->ift_cleaned, txq->ift_ctx->ifc_softc_ctx.isc_tx_nsegments,
2800 			       reclaim, thresh);
2801 
2802 		}
2803 #endif
2804 		return (0);
2805 	}
2806 	iflib_tx_desc_free(txq, reclaim);
2807 	txq->ift_cleaned += reclaim;
2808 	txq->ift_in_use -= reclaim;
2809 
2810 	if (txq->ift_active == FALSE)
2811 		txq->ift_active = TRUE;
2812 
2813 	return (reclaim);
2814 }
2815 
2816 static struct mbuf **
2817 _ring_peek_one(struct ifmp_ring *r, int cidx, int offset)
2818 {
2819 
2820 	return (__DEVOLATILE(struct mbuf **, &r->items[(cidx + offset) & (r->size-1)]));
2821 }
2822 
2823 static void
2824 iflib_txq_check_drain(iflib_txq_t txq, int budget)
2825 {
2826 
2827 	ifmp_ring_check_drainage(txq->ift_br[0], budget);
2828 }
2829 
2830 static uint32_t
2831 iflib_txq_can_drain(struct ifmp_ring *r)
2832 {
2833 	iflib_txq_t txq = r->cookie;
2834 	if_ctx_t ctx = txq->ift_ctx;
2835 
2836 	return ((TXQ_AVAIL(txq) >= MAX_TX_DESC(ctx)) ||
2837 		ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, txq->ift_cidx_processed, false));
2838 }
2839 
2840 static uint32_t
2841 iflib_txq_drain(struct ifmp_ring *r, uint32_t cidx, uint32_t pidx)
2842 {
2843 	iflib_txq_t txq = r->cookie;
2844 	if_ctx_t ctx = txq->ift_ctx;
2845 	if_t ifp = ctx->ifc_ifp;
2846 	struct mbuf **mp, *m;
2847 	int i, count, consumed, pkt_sent, bytes_sent, mcast_sent, avail, err, in_use_prev, desc_used;
2848 
2849 	if (__predict_false(!(if_getdrvflags(ifp) & IFF_DRV_RUNNING) ||
2850 			    !LINK_ACTIVE(ctx))) {
2851 		DBG_COUNTER_INC(txq_drain_notready);
2852 		return (0);
2853 	}
2854 
2855 	avail = IDXDIFF(pidx, cidx, r->size);
2856 	if (__predict_false(ctx->ifc_flags & IFC_QFLUSH)) {
2857 		DBG_COUNTER_INC(txq_drain_flushing);
2858 		for (i = 0; i < avail; i++) {
2859 			m_freem(r->items[(cidx + i) & (r->size-1)]);
2860 			r->items[(cidx + i) & (r->size-1)] = NULL;
2861 		}
2862 		return (avail);
2863 	}
2864 	iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx));
2865 	if (__predict_false(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE)) {
2866 		txq->ift_qstatus = IFLIB_QUEUE_IDLE;
2867 		CALLOUT_LOCK(txq);
2868 		callout_stop(&txq->ift_timer);
2869 		callout_stop(&txq->ift_db_check);
2870 		CALLOUT_UNLOCK(txq);
2871 		DBG_COUNTER_INC(txq_drain_oactive);
2872 		return (0);
2873 	}
2874 	consumed = mcast_sent = bytes_sent = pkt_sent = 0;
2875 	count = MIN(avail, TX_BATCH_SIZE);
2876 
2877 	for (desc_used = i = 0; i < count && TXQ_AVAIL(txq) > MAX_TX_DESC(ctx) + 2; i++) {
2878 		mp = _ring_peek_one(r, cidx, i);
2879 		in_use_prev = txq->ift_in_use;
2880 		err = iflib_encap(txq, mp);
2881 		/*
2882 		 * What other errors should we bail out for?
2883 		 */
2884 		if (err == ENOBUFS) {
2885 			DBG_COUNTER_INC(txq_drain_encapfail);
2886 			break;
2887 		}
2888 		consumed++;
2889 		if (err)
2890 			continue;
2891 
2892 		pkt_sent++;
2893 		m = *mp;
2894 		DBG_COUNTER_INC(tx_sent);
2895 		bytes_sent += m->m_pkthdr.len;
2896 		if (m->m_flags & M_MCAST)
2897 			mcast_sent++;
2898 
2899 		txq->ift_db_pending += (txq->ift_in_use - in_use_prev);
2900 		desc_used += (txq->ift_in_use - in_use_prev);
2901 		iflib_txd_db_check(ctx, txq, FALSE);
2902 		ETHER_BPF_MTAP(ifp, m);
2903 		if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)))
2904 			break;
2905 
2906 		if (desc_used > TXQ_MAX_DB_CONSUMED(ctx))
2907 			break;
2908 	}
2909 
2910 	if ((iflib_min_tx_latency || iflib_txq_min_occupancy(txq)) && txq->ift_db_pending)
2911 		iflib_txd_db_check(ctx, txq, TRUE);
2912 	else if ((txq->ift_db_pending || TXQ_AVAIL(txq) < MAX_TX_DESC(ctx)) &&
2913 		 (callout_pending(&txq->ift_db_check) == 0)) {
2914 		txq->ift_db_pending_queued = txq->ift_db_pending;
2915 		callout_reset_on(&txq->ift_db_check, 1, iflib_txd_deferred_db_check,
2916 				 txq, txq->ift_db_check.c_cpu);
2917 	}
2918 	if_inc_counter(ifp, IFCOUNTER_OBYTES, bytes_sent);
2919 	if_inc_counter(ifp, IFCOUNTER_OPACKETS, pkt_sent);
2920 	if (mcast_sent)
2921 		if_inc_counter(ifp, IFCOUNTER_OMCASTS, mcast_sent);
2922 
2923 	return (consumed);
2924 }
2925 
2926 static void
2927 _task_fn_tx(void *context, int pending)
2928 {
2929 	iflib_txq_t txq = context;
2930 	if_ctx_t ctx = txq->ift_ctx;
2931 
2932 	if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
2933 		return;
2934 	ifmp_ring_check_drainage(txq->ift_br[0], TX_BATCH_SIZE);
2935 }
2936 
2937 static void
2938 _task_fn_rx(void *context, int pending)
2939 {
2940 	iflib_rxq_t rxq = context;
2941 	if_ctx_t ctx = rxq->ifr_ctx;
2942 	bool more;
2943 
2944 	DBG_COUNTER_INC(task_fn_rxs);
2945 	if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)))
2946 		return;
2947 
2948 	if ((more = iflib_rxeof(rxq, 16 /* XXX */)) == false) {
2949 		if (ctx->ifc_flags & IFC_LEGACY)
2950 			IFDI_INTR_ENABLE(ctx);
2951 		else {
2952 			DBG_COUNTER_INC(rx_intr_enables);
2953 			IFDI_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id);
2954 		}
2955 	}
2956 	if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)))
2957 		return;
2958 	if (more)
2959 		GROUPTASK_ENQUEUE(&rxq->ifr_task);
2960 }
2961 
2962 static void
2963 _task_fn_admin(void *context, int pending)
2964 {
2965 	if_ctx_t ctx = context;
2966 	if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
2967 	iflib_txq_t txq;
2968 	int i;
2969 
2970 	if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
2971 		return;
2972 
2973 	CTX_LOCK(ctx);
2974 	for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) {
2975 		CALLOUT_LOCK(txq);
2976 		callout_stop(&txq->ift_timer);
2977 		CALLOUT_UNLOCK(txq);
2978 	}
2979 	IFDI_UPDATE_ADMIN_STATUS(ctx);
2980 	for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++)
2981 		callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, txq->ift_timer.c_cpu);
2982 	IFDI_LINK_INTR_ENABLE(ctx);
2983 	CTX_UNLOCK(ctx);
2984 
2985 	if (LINK_ACTIVE(ctx) == 0)
2986 		return;
2987 	for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++)
2988 		iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET);
2989 }
2990 
2991 
2992 static void
2993 _task_fn_iov(void *context, int pending)
2994 {
2995 	if_ctx_t ctx = context;
2996 
2997 	if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
2998 		return;
2999 
3000 	CTX_LOCK(ctx);
3001 	IFDI_VFLR_HANDLE(ctx);
3002 	CTX_UNLOCK(ctx);
3003 }
3004 
3005 static int
3006 iflib_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
3007 {
3008 	int err;
3009 	if_int_delay_info_t info;
3010 	if_ctx_t ctx;
3011 
3012 	info = (if_int_delay_info_t)arg1;
3013 	ctx = info->iidi_ctx;
3014 	info->iidi_req = req;
3015 	info->iidi_oidp = oidp;
3016 	CTX_LOCK(ctx);
3017 	err = IFDI_SYSCTL_INT_DELAY(ctx, info);
3018 	CTX_UNLOCK(ctx);
3019 	return (err);
3020 }
3021 
3022 /*********************************************************************
3023  *
3024  *  IFNET FUNCTIONS
3025  *
3026  **********************************************************************/
3027 
3028 static void
3029 iflib_if_init_locked(if_ctx_t ctx)
3030 {
3031 	iflib_stop(ctx);
3032 	iflib_init_locked(ctx);
3033 }
3034 
3035 
3036 static void
3037 iflib_if_init(void *arg)
3038 {
3039 	if_ctx_t ctx = arg;
3040 
3041 	CTX_LOCK(ctx);
3042 	iflib_if_init_locked(ctx);
3043 	CTX_UNLOCK(ctx);
3044 }
3045 
3046 static int
3047 iflib_if_transmit(if_t ifp, struct mbuf *m)
3048 {
3049 	if_ctx_t	ctx = if_getsoftc(ifp);
3050 
3051 	iflib_txq_t txq;
3052 	struct mbuf *marr[8], **mp, *next;
3053 	int err, i, count, qidx;
3054 
3055 	if (__predict_false((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || !LINK_ACTIVE(ctx))) {
3056 		DBG_COUNTER_INC(tx_frees);
3057 		m_freem(m);
3058 		return (0);
3059 	}
3060 
3061 	qidx = 0;
3062 	if ((NTXQSETS(ctx) > 1) && M_HASHTYPE_GET(m))
3063 		qidx = QIDX(ctx, m);
3064 	/*
3065 	 * XXX calculate buf_ring based on flowid (divvy up bits?)
3066 	 */
3067 	txq = &ctx->ifc_txqs[qidx];
3068 
3069 #ifdef DRIVER_BACKPRESSURE
3070 	if (txq->ift_closed) {
3071 		while (m != NULL) {
3072 			next = m->m_nextpkt;
3073 			m->m_nextpkt = NULL;
3074 			m_freem(m);
3075 			m = next;
3076 		}
3077 		return (ENOBUFS);
3078 	}
3079 #endif
3080 	qidx = count = 0;
3081 	mp = marr;
3082 	next = m;
3083 	do {
3084 		count++;
3085 		next = next->m_nextpkt;
3086 	} while (next != NULL);
3087 
3088 	if (count > nitems(marr))
3089 		if ((mp = malloc(count*sizeof(struct mbuf *), M_IFLIB, M_NOWAIT)) == NULL) {
3090 			/* XXX check nextpkt */
3091 			m_freem(m);
3092 			/* XXX simplify for now */
3093 			DBG_COUNTER_INC(tx_frees);
3094 			return (ENOBUFS);
3095 		}
3096 	for (next = m, i = 0; next != NULL; i++) {
3097 		mp[i] = next;
3098 		next = next->m_nextpkt;
3099 		mp[i]->m_nextpkt = NULL;
3100 	}
3101 	DBG_COUNTER_INC(tx_seen);
3102 	err = ifmp_ring_enqueue(txq->ift_br[0], (void **)mp, count, TX_BATCH_SIZE);
3103 
3104 	if (iflib_txq_can_drain(txq->ift_br[0]))
3105 		GROUPTASK_ENQUEUE(&txq->ift_task);
3106 	if (err) {
3107 		/* support forthcoming later */
3108 #ifdef DRIVER_BACKPRESSURE
3109 		txq->ift_closed = TRUE;
3110 #endif
3111 		for (i = 0; i < count; i++)
3112 			m_freem(mp[i]);
3113 		ifmp_ring_check_drainage(txq->ift_br[0], TX_BATCH_SIZE);
3114 	}
3115 	if (count > nitems(marr))
3116 		free(mp, M_IFLIB);
3117 
3118 	return (err);
3119 }
3120 
3121 static void
3122 iflib_if_qflush(if_t ifp)
3123 {
3124 	if_ctx_t ctx = if_getsoftc(ifp);
3125 	iflib_txq_t txq = ctx->ifc_txqs;
3126 	int i;
3127 
3128 	CTX_LOCK(ctx);
3129 	ctx->ifc_flags |= IFC_QFLUSH;
3130 	CTX_UNLOCK(ctx);
3131 	for (i = 0; i < NTXQSETS(ctx); i++, txq++)
3132 		while (!(ifmp_ring_is_idle(txq->ift_br[0]) || ifmp_ring_is_stalled(txq->ift_br[0])))
3133 			iflib_txq_check_drain(txq, 0);
3134 	CTX_LOCK(ctx);
3135 	ctx->ifc_flags &= ~IFC_QFLUSH;
3136 	CTX_UNLOCK(ctx);
3137 
3138 	if_qflush(ifp);
3139 }
3140 
3141 #define IFCAP_REINIT (IFCAP_HWCSUM|IFCAP_TSO4|IFCAP_TSO6|IFCAP_VLAN_HWTAGGING|IFCAP_VLAN_MTU | \
3142 		      IFCAP_VLAN_HWFILTER | IFCAP_VLAN_HWTSO)
3143 
3144 #define IFCAP_FLAGS (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_HWCSUM | IFCAP_LRO | \
3145 		     IFCAP_TSO4 | IFCAP_TSO6 | IFCAP_VLAN_HWTAGGING |	\
3146 		     IFCAP_VLAN_MTU | IFCAP_VLAN_HWFILTER | IFCAP_VLAN_HWTSO)
3147 
3148 static int
3149 iflib_if_ioctl(if_t ifp, u_long command, caddr_t data)
3150 {
3151 	if_ctx_t ctx = if_getsoftc(ifp);
3152 	struct ifreq	*ifr = (struct ifreq *)data;
3153 #if defined(INET) || defined(INET6)
3154 	struct ifaddr	*ifa = (struct ifaddr *)data;
3155 #endif
3156 	bool		avoid_reset = FALSE;
3157 	int		err = 0, reinit = 0, bits;
3158 
3159 	switch (command) {
3160 	case SIOCSIFADDR:
3161 #ifdef INET
3162 		if (ifa->ifa_addr->sa_family == AF_INET)
3163 			avoid_reset = TRUE;
3164 #endif
3165 #ifdef INET6
3166 		if (ifa->ifa_addr->sa_family == AF_INET6)
3167 			avoid_reset = TRUE;
3168 #endif
3169 		/*
3170 		** Calling init results in link renegotiation,
3171 		** so we avoid doing it when possible.
3172 		*/
3173 		if (avoid_reset) {
3174 			if_setflagbits(ifp, IFF_UP,0);
3175 			if (!(if_getdrvflags(ifp)& IFF_DRV_RUNNING))
3176 				reinit = 1;
3177 #ifdef INET
3178 			if (!(if_getflags(ifp) & IFF_NOARP))
3179 				arp_ifinit(ifp, ifa);
3180 #endif
3181 		} else
3182 			err = ether_ioctl(ifp, command, data);
3183 		break;
3184 	case SIOCSIFMTU:
3185 		CTX_LOCK(ctx);
3186 		if (ifr->ifr_mtu == if_getmtu(ifp)) {
3187 			CTX_UNLOCK(ctx);
3188 			break;
3189 		}
3190 		bits = if_getdrvflags(ifp);
3191 		/* stop the driver and free any clusters before proceeding */
3192 		iflib_stop(ctx);
3193 
3194 		if ((err = IFDI_MTU_SET(ctx, ifr->ifr_mtu)) == 0) {
3195 			if (ifr->ifr_mtu > ctx->ifc_max_fl_buf_size)
3196 				ctx->ifc_flags |= IFC_MULTISEG;
3197 			else
3198 				ctx->ifc_flags &= ~IFC_MULTISEG;
3199 			err = if_setmtu(ifp, ifr->ifr_mtu);
3200 		}
3201 		iflib_init_locked(ctx);
3202 		if_setdrvflags(ifp, bits);
3203 		CTX_UNLOCK(ctx);
3204 		break;
3205 	case SIOCSIFFLAGS:
3206 		CTX_LOCK(ctx);
3207 		if (if_getflags(ifp) & IFF_UP) {
3208 			if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
3209 				if ((if_getflags(ifp) ^ ctx->ifc_if_flags) &
3210 				    (IFF_PROMISC | IFF_ALLMULTI)) {
3211 					err = IFDI_PROMISC_SET(ctx, if_getflags(ifp));
3212 				}
3213 			} else
3214 				reinit = 1;
3215 		} else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
3216 			iflib_stop(ctx);
3217 		}
3218 		ctx->ifc_if_flags = if_getflags(ifp);
3219 		CTX_UNLOCK(ctx);
3220 		break;
3221 
3222 		break;
3223 	case SIOCADDMULTI:
3224 	case SIOCDELMULTI:
3225 		if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
3226 			CTX_LOCK(ctx);
3227 			IFDI_INTR_DISABLE(ctx);
3228 			IFDI_MULTI_SET(ctx);
3229 			IFDI_INTR_ENABLE(ctx);
3230 			CTX_UNLOCK(ctx);
3231 		}
3232 		break;
3233 	case SIOCSIFMEDIA:
3234 		CTX_LOCK(ctx);
3235 		IFDI_MEDIA_SET(ctx);
3236 		CTX_UNLOCK(ctx);
3237 		/* falls thru */
3238 	case SIOCGIFMEDIA:
3239 		err = ifmedia_ioctl(ifp, ifr, &ctx->ifc_media, command);
3240 		break;
3241 	case SIOCGI2C:
3242 	{
3243 		struct ifi2creq i2c;
3244 
3245 		err = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
3246 		if (err != 0)
3247 			break;
3248 		if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
3249 			err = EINVAL;
3250 			break;
3251 		}
3252 		if (i2c.len > sizeof(i2c.data)) {
3253 			err = EINVAL;
3254 			break;
3255 		}
3256 
3257 		if ((err = IFDI_I2C_REQ(ctx, &i2c)) == 0)
3258 			err = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
3259 		break;
3260 	}
3261 	case SIOCSIFCAP:
3262 	{
3263 		int mask, setmask;
3264 
3265 		mask = ifr->ifr_reqcap ^ if_getcapenable(ifp);
3266 		setmask = 0;
3267 #ifdef TCP_OFFLOAD
3268 		setmask |= mask & (IFCAP_TOE4|IFCAP_TOE6);
3269 #endif
3270 		setmask |= (mask & IFCAP_FLAGS);
3271 
3272 		if ((mask & IFCAP_WOL) &&
3273 		    (if_getcapabilities(ifp) & IFCAP_WOL) != 0)
3274 			setmask |= (mask & (IFCAP_WOL_MCAST|IFCAP_WOL_MAGIC));
3275 		if_vlancap(ifp);
3276 		/*
3277 		 * want to ensure that traffic has stopped before we change any of the flags
3278 		 */
3279 		if (setmask) {
3280 			CTX_LOCK(ctx);
3281 			bits = if_getdrvflags(ifp);
3282 			if (setmask & IFCAP_REINIT)
3283 				iflib_stop(ctx);
3284 			if_togglecapenable(ifp, setmask);
3285 			if (setmask & IFCAP_REINIT)
3286 				iflib_init_locked(ctx);
3287 			if_setdrvflags(ifp, bits);
3288 			CTX_UNLOCK(ctx);
3289 		}
3290 		break;
3291 	    }
3292 	case SIOCGPRIVATE_0:
3293 	case SIOCSDRVSPEC:
3294 	case SIOCGDRVSPEC:
3295 		CTX_LOCK(ctx);
3296 		err = IFDI_PRIV_IOCTL(ctx, command, data);
3297 		CTX_UNLOCK(ctx);
3298 		break;
3299 	default:
3300 		err = ether_ioctl(ifp, command, data);
3301 		break;
3302 	}
3303 	if (reinit)
3304 		iflib_if_init(ctx);
3305 	return (err);
3306 }
3307 
3308 static uint64_t
3309 iflib_if_get_counter(if_t ifp, ift_counter cnt)
3310 {
3311 	if_ctx_t ctx = if_getsoftc(ifp);
3312 
3313 	return (IFDI_GET_COUNTER(ctx, cnt));
3314 }
3315 
3316 /*********************************************************************
3317  *
3318  *  OTHER FUNCTIONS EXPORTED TO THE STACK
3319  *
3320  **********************************************************************/
3321 
3322 static void
3323 iflib_vlan_register(void *arg, if_t ifp, uint16_t vtag)
3324 {
3325 	if_ctx_t ctx = if_getsoftc(ifp);
3326 
3327 	if ((void *)ctx != arg)
3328 		return;
3329 
3330 	if ((vtag == 0) || (vtag > 4095))
3331 		return;
3332 
3333 	CTX_LOCK(ctx);
3334 	IFDI_VLAN_REGISTER(ctx, vtag);
3335 	/* Re-init to load the changes */
3336 	if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
3337 		iflib_init_locked(ctx);
3338 	CTX_UNLOCK(ctx);
3339 }
3340 
3341 static void
3342 iflib_vlan_unregister(void *arg, if_t ifp, uint16_t vtag)
3343 {
3344 	if_ctx_t ctx = if_getsoftc(ifp);
3345 
3346 	if ((void *)ctx != arg)
3347 		return;
3348 
3349 	if ((vtag == 0) || (vtag > 4095))
3350 		return;
3351 
3352 	CTX_LOCK(ctx);
3353 	IFDI_VLAN_UNREGISTER(ctx, vtag);
3354 	/* Re-init to load the changes */
3355 	if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
3356 		iflib_init_locked(ctx);
3357 	CTX_UNLOCK(ctx);
3358 }
3359 
3360 static void
3361 iflib_led_func(void *arg, int onoff)
3362 {
3363 	if_ctx_t ctx = arg;
3364 
3365 	CTX_LOCK(ctx);
3366 	IFDI_LED_FUNC(ctx, onoff);
3367 	CTX_UNLOCK(ctx);
3368 }
3369 
3370 /*********************************************************************
3371  *
3372  *  BUS FUNCTION DEFINITIONS
3373  *
3374  **********************************************************************/
3375 
3376 int
3377 iflib_device_probe(device_t dev)
3378 {
3379 	pci_vendor_info_t *ent;
3380 
3381 	uint16_t	pci_vendor_id, pci_device_id;
3382 	uint16_t	pci_subvendor_id, pci_subdevice_id;
3383 	uint16_t	pci_rev_id;
3384 	if_shared_ctx_t sctx;
3385 
3386 	if ((sctx = DEVICE_REGISTER(dev)) == NULL || sctx->isc_magic != IFLIB_MAGIC)
3387 		return (ENOTSUP);
3388 
3389 	pci_vendor_id = pci_get_vendor(dev);
3390 	pci_device_id = pci_get_device(dev);
3391 	pci_subvendor_id = pci_get_subvendor(dev);
3392 	pci_subdevice_id = pci_get_subdevice(dev);
3393 	pci_rev_id = pci_get_revid(dev);
3394 	if (sctx->isc_parse_devinfo != NULL)
3395 		sctx->isc_parse_devinfo(&pci_device_id, &pci_subvendor_id, &pci_subdevice_id, &pci_rev_id);
3396 
3397 	ent = sctx->isc_vendor_info;
3398 	while (ent->pvi_vendor_id != 0) {
3399 		if (pci_vendor_id != ent->pvi_vendor_id) {
3400 			ent++;
3401 			continue;
3402 		}
3403 		if ((pci_device_id == ent->pvi_device_id) &&
3404 		    ((pci_subvendor_id == ent->pvi_subvendor_id) ||
3405 		     (ent->pvi_subvendor_id == 0)) &&
3406 		    ((pci_subdevice_id == ent->pvi_subdevice_id) ||
3407 		     (ent->pvi_subdevice_id == 0)) &&
3408 		    ((pci_rev_id == ent->pvi_rev_id) ||
3409 		     (ent->pvi_rev_id == 0))) {
3410 
3411 			device_set_desc_copy(dev, ent->pvi_name);
3412 			/* this needs to be changed to zero if the bus probing code
3413 			 * ever stops re-probing on best match because the sctx
3414 			 * may have its values over written by register calls
3415 			 * in subsequent probes
3416 			 */
3417 			return (BUS_PROBE_DEFAULT);
3418 		}
3419 		ent++;
3420 	}
3421 	return (ENXIO);
3422 }
3423 
3424 int
3425 iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ctxp)
3426 {
3427 	int err, rid, msix, msix_bar;
3428 	if_ctx_t ctx;
3429 	if_t ifp;
3430 	if_softc_ctx_t scctx;
3431 
3432 
3433 	ctx = malloc(sizeof(* ctx), M_IFLIB, M_WAITOK|M_ZERO);
3434 
3435 	if (sc == NULL) {
3436 		sc = malloc(sctx->isc_driver->size, M_IFLIB, M_WAITOK|M_ZERO);
3437 		device_set_softc(dev, ctx);
3438 	}
3439 
3440 	ctx->ifc_sctx = sctx;
3441 	ctx->ifc_dev = dev;
3442 	ctx->ifc_txrx = *sctx->isc_txrx;
3443 	ctx->ifc_softc = sc;
3444 
3445 	if ((err = iflib_register(ctx)) != 0) {
3446 		device_printf(dev, "iflib_register failed %d\n", err);
3447 		return (err);
3448 	}
3449 	iflib_add_device_sysctl_pre(ctx);
3450 	if ((err = IFDI_ATTACH_PRE(ctx)) != 0) {
3451 		device_printf(dev, "IFDI_ATTACH_PRE failed %d\n", err);
3452 		return (err);
3453 	}
3454 #ifdef ACPI_DMAR
3455 	if (dmar_get_dma_tag(device_get_parent(dev), dev) != NULL)
3456 		ctx->ifc_flags |= IFC_DMAR;
3457 #endif
3458 
3459 	scctx = &ctx->ifc_softc_ctx;
3460 	msix_bar = scctx->isc_msix_bar;
3461 
3462 	if (scctx->isc_tx_nsegments > sctx->isc_ntxd / MAX_SINGLE_PACKET_FRACTION)
3463 		scctx->isc_tx_nsegments = max(1, sctx->isc_ntxd / MAX_SINGLE_PACKET_FRACTION);
3464 	if (scctx->isc_tx_tso_segments_max > sctx->isc_ntxd / MAX_SINGLE_PACKET_FRACTION)
3465 		scctx->isc_tx_tso_segments_max = max(1, sctx->isc_ntxd / MAX_SINGLE_PACKET_FRACTION);
3466 
3467 	ifp = ctx->ifc_ifp;
3468 
3469 	/*
3470 	 * XXX sanity check that ntxd & nrxd are a power of 2
3471 	 */
3472 
3473 	/*
3474 	 * Protect the stack against modern hardware
3475 	 */
3476 	if (scctx->isc_tx_tso_size_max > FREEBSD_TSO_SIZE_MAX)
3477 		scctx->isc_tx_tso_size_max = FREEBSD_TSO_SIZE_MAX;
3478 
3479 	/* TSO parameters - dig these out of the data sheet - simply correspond to tag setup */
3480 	ifp->if_hw_tsomaxsegcount = scctx->isc_tx_tso_segments_max;
3481 	ifp->if_hw_tsomax = scctx->isc_tx_tso_size_max;
3482 	ifp->if_hw_tsomaxsegsize = scctx->isc_tx_tso_segsize_max;
3483 	if (scctx->isc_rss_table_size == 0)
3484 		scctx->isc_rss_table_size = 64;
3485 	scctx->isc_rss_table_mask = scctx->isc_rss_table_size-1;;
3486 	/*
3487 	** Now setup MSI or MSI/X, should
3488 	** return us the number of supported
3489 	** vectors. (Will be 1 for MSI)
3490 	*/
3491 	if (sctx->isc_flags & IFLIB_SKIP_MSIX) {
3492 		msix = scctx->isc_vectors;
3493 	} else if (scctx->isc_msix_bar != 0)
3494 		msix = iflib_msix_init(ctx);
3495 	else {
3496 		scctx->isc_vectors = 1;
3497 		scctx->isc_ntxqsets = 1;
3498 		scctx->isc_nrxqsets = 1;
3499 		scctx->isc_intr = IFLIB_INTR_LEGACY;
3500 		msix = 0;
3501 	}
3502 	/* Get memory for the station queues */
3503 	if ((err = iflib_queues_alloc(ctx))) {
3504 		device_printf(dev, "Unable to allocate queue memory\n");
3505 		goto fail;
3506 	}
3507 
3508 	if ((err = iflib_qset_structures_setup(ctx))) {
3509 		device_printf(dev, "qset structure setup failed %d\n", err);
3510 		goto fail_queues;
3511 	}
3512 
3513 	if (msix > 1 && (err = IFDI_MSIX_INTR_ASSIGN(ctx, msix)) != 0) {
3514 		device_printf(dev, "IFDI_MSIX_INTR_ASSIGN failed %d\n", err);
3515 		goto fail_intr_free;
3516 	}
3517 	if (msix <= 1) {
3518 		rid = 0;
3519 		if (scctx->isc_intr == IFLIB_INTR_MSI) {
3520 			MPASS(msix == 1);
3521 			rid = 1;
3522 		}
3523 		if ((err = iflib_legacy_setup(ctx, ctx->isc_legacy_intr, ctx, &rid, "irq0")) != 0) {
3524 			device_printf(dev, "iflib_legacy_setup failed %d\n", err);
3525 			goto fail_intr_free;
3526 		}
3527 	}
3528 	ether_ifattach(ctx->ifc_ifp, ctx->ifc_mac);
3529 	if ((err = IFDI_ATTACH_POST(ctx)) != 0) {
3530 		device_printf(dev, "IFDI_ATTACH_POST failed %d\n", err);
3531 		goto fail_detach;
3532 	}
3533 	if ((err = iflib_netmap_attach(ctx))) {
3534 		device_printf(ctx->ifc_dev, "netmap attach failed: %d\n", err);
3535 		goto fail_detach;
3536 	}
3537 	*ctxp = ctx;
3538 
3539 	iflib_add_device_sysctl_post(ctx);
3540 	return (0);
3541 fail_detach:
3542 	ether_ifdetach(ctx->ifc_ifp);
3543 fail_intr_free:
3544 	if (scctx->isc_intr == IFLIB_INTR_MSIX || scctx->isc_intr == IFLIB_INTR_MSI)
3545 		pci_release_msi(ctx->ifc_dev);
3546 fail_queues:
3547 	/* XXX free queues */
3548 fail:
3549 	IFDI_DETACH(ctx);
3550 	return (err);
3551 }
3552 
3553 int
3554 iflib_device_attach(device_t dev)
3555 {
3556 	if_ctx_t ctx;
3557 	if_shared_ctx_t sctx;
3558 
3559 	if ((sctx = DEVICE_REGISTER(dev)) == NULL || sctx->isc_magic != IFLIB_MAGIC)
3560 		return (ENOTSUP);
3561 
3562 	pci_enable_busmaster(dev);
3563 
3564 	return (iflib_device_register(dev, NULL, sctx, &ctx));
3565 }
3566 
3567 int
3568 iflib_device_deregister(if_ctx_t ctx)
3569 {
3570 	if_t ifp = ctx->ifc_ifp;
3571 	iflib_txq_t txq;
3572 	iflib_rxq_t rxq;
3573 	device_t dev = ctx->ifc_dev;
3574 	int i;
3575 	struct taskqgroup *tqg;
3576 
3577 	/* Make sure VLANS are not using driver */
3578 	if (if_vlantrunkinuse(ifp)) {
3579 		device_printf(dev,"Vlan in use, detach first\n");
3580 		return (EBUSY);
3581 	}
3582 
3583 	CTX_LOCK(ctx);
3584 	ctx->ifc_in_detach = 1;
3585 	iflib_stop(ctx);
3586 	CTX_UNLOCK(ctx);
3587 
3588 	/* Unregister VLAN events */
3589 	if (ctx->ifc_vlan_attach_event != NULL)
3590 		EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event);
3591 	if (ctx->ifc_vlan_detach_event != NULL)
3592 		EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event);
3593 
3594 	iflib_netmap_detach(ifp);
3595 	ether_ifdetach(ifp);
3596 	/* ether_ifdetach calls if_qflush - lock must be destroy afterwards*/
3597 	CTX_LOCK_DESTROY(ctx);
3598 	if (ctx->ifc_led_dev != NULL)
3599 		led_destroy(ctx->ifc_led_dev);
3600 	/* XXX drain any dependent tasks */
3601 	tqg = qgroup_if_io_tqg;
3602 	for (txq = ctx->ifc_txqs, i = 0, rxq = ctx->ifc_rxqs; i < NTXQSETS(ctx); i++, txq++) {
3603 		callout_drain(&txq->ift_timer);
3604 		callout_drain(&txq->ift_db_check);
3605 		if (txq->ift_task.gt_uniq != NULL)
3606 			taskqgroup_detach(tqg, &txq->ift_task);
3607 	}
3608 	for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) {
3609 		if (rxq->ifr_task.gt_uniq != NULL)
3610 			taskqgroup_detach(tqg, &rxq->ifr_task);
3611 	}
3612 	tqg = qgroup_if_config_tqg;
3613 	if (ctx->ifc_admin_task.gt_uniq != NULL)
3614 		taskqgroup_detach(tqg, &ctx->ifc_admin_task);
3615 	if (ctx->ifc_vflr_task.gt_uniq != NULL)
3616 		taskqgroup_detach(tqg, &ctx->ifc_vflr_task);
3617 
3618 	IFDI_DETACH(ctx);
3619 	if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_LEGACY) {
3620 		pci_release_msi(dev);
3621 	}
3622 	if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_MSIX) {
3623 		iflib_irq_free(ctx, &ctx->ifc_legacy_irq);
3624 	}
3625 	if (ctx->ifc_msix_mem != NULL) {
3626 		bus_release_resource(ctx->ifc_dev, SYS_RES_MEMORY,
3627 			ctx->ifc_softc_ctx.isc_msix_bar, ctx->ifc_msix_mem);
3628 		ctx->ifc_msix_mem = NULL;
3629 	}
3630 
3631 	bus_generic_detach(dev);
3632 	if_free(ifp);
3633 
3634 	iflib_tx_structures_free(ctx);
3635 	iflib_rx_structures_free(ctx);
3636 	return (0);
3637 }
3638 
3639 
3640 int
3641 iflib_device_detach(device_t dev)
3642 {
3643 	if_ctx_t ctx = device_get_softc(dev);
3644 
3645 	return (iflib_device_deregister(ctx));
3646 }
3647 
3648 int
3649 iflib_device_suspend(device_t dev)
3650 {
3651 	if_ctx_t ctx = device_get_softc(dev);
3652 
3653 	CTX_LOCK(ctx);
3654 	IFDI_SUSPEND(ctx);
3655 	CTX_UNLOCK(ctx);
3656 
3657 	return bus_generic_suspend(dev);
3658 }
3659 int
3660 iflib_device_shutdown(device_t dev)
3661 {
3662 	if_ctx_t ctx = device_get_softc(dev);
3663 
3664 	CTX_LOCK(ctx);
3665 	IFDI_SHUTDOWN(ctx);
3666 	CTX_UNLOCK(ctx);
3667 
3668 	return bus_generic_suspend(dev);
3669 }
3670 
3671 
3672 int
3673 iflib_device_resume(device_t dev)
3674 {
3675 	if_ctx_t ctx = device_get_softc(dev);
3676 	iflib_txq_t txq = ctx->ifc_txqs;
3677 
3678 	CTX_LOCK(ctx);
3679 	IFDI_RESUME(ctx);
3680 	iflib_init_locked(ctx);
3681 	CTX_UNLOCK(ctx);
3682 	for (int i = 0; i < NTXQSETS(ctx); i++, txq++)
3683 		iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET);
3684 
3685 	return (bus_generic_resume(dev));
3686 }
3687 
3688 int
3689 iflib_device_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params)
3690 {
3691 	int error;
3692 	if_ctx_t ctx = device_get_softc(dev);
3693 
3694 	CTX_LOCK(ctx);
3695 	error = IFDI_IOV_INIT(ctx, num_vfs, params);
3696 	CTX_UNLOCK(ctx);
3697 
3698 	return (error);
3699 }
3700 
3701 void
3702 iflib_device_iov_uninit(device_t dev)
3703 {
3704 	if_ctx_t ctx = device_get_softc(dev);
3705 
3706 	CTX_LOCK(ctx);
3707 	IFDI_IOV_UNINIT(ctx);
3708 	CTX_UNLOCK(ctx);
3709 }
3710 
3711 int
3712 iflib_device_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params)
3713 {
3714 	int error;
3715 	if_ctx_t ctx = device_get_softc(dev);
3716 
3717 	CTX_LOCK(ctx);
3718 	error = IFDI_IOV_VF_ADD(ctx, vfnum, params);
3719 	CTX_UNLOCK(ctx);
3720 
3721 	return (error);
3722 }
3723 
3724 /*********************************************************************
3725  *
3726  *  MODULE FUNCTION DEFINITIONS
3727  *
3728  **********************************************************************/
3729 
3730 /*
3731  * - Start a fast taskqueue thread for each core
3732  * - Start a taskqueue for control operations
3733  */
3734 static int
3735 iflib_module_init(void)
3736 {
3737 	return (0);
3738 }
3739 
3740 static int
3741 iflib_module_event_handler(module_t mod, int what, void *arg)
3742 {
3743 	int err;
3744 
3745 	switch (what) {
3746 	case MOD_LOAD:
3747 		if ((err = iflib_module_init()) != 0)
3748 			return (err);
3749 		break;
3750 	case MOD_UNLOAD:
3751 		return (EBUSY);
3752 	default:
3753 		return (EOPNOTSUPP);
3754 	}
3755 
3756 	return (0);
3757 }
3758 
3759 /*********************************************************************
3760  *
3761  *  PUBLIC FUNCTION DEFINITIONS
3762  *     ordered as in iflib.h
3763  *
3764  **********************************************************************/
3765 
3766 
3767 static void
3768 _iflib_assert(if_shared_ctx_t sctx)
3769 {
3770 	MPASS(sctx->isc_tx_maxsize);
3771 	MPASS(sctx->isc_tx_maxsegsize);
3772 
3773 	MPASS(sctx->isc_rx_maxsize);
3774 	MPASS(sctx->isc_rx_nsegments);
3775 	MPASS(sctx->isc_rx_maxsegsize);
3776 
3777 
3778 	MPASS(sctx->isc_txrx->ift_txd_encap);
3779 	MPASS(sctx->isc_txrx->ift_txd_flush);
3780 	MPASS(sctx->isc_txrx->ift_txd_credits_update);
3781 	MPASS(sctx->isc_txrx->ift_rxd_available);
3782 	MPASS(sctx->isc_txrx->ift_rxd_pkt_get);
3783 	MPASS(sctx->isc_txrx->ift_rxd_refill);
3784 	MPASS(sctx->isc_txrx->ift_rxd_flush);
3785 	MPASS(sctx->isc_nrxd);
3786 }
3787 
3788 static int
3789 iflib_register(if_ctx_t ctx)
3790 {
3791 	if_shared_ctx_t sctx = ctx->ifc_sctx;
3792 	driver_t *driver = sctx->isc_driver;
3793 	device_t dev = ctx->ifc_dev;
3794 	if_t ifp;
3795 
3796 	_iflib_assert(sctx);
3797 
3798 	CTX_LOCK_INIT(ctx, device_get_nameunit(ctx->ifc_dev));
3799 	MPASS(ctx->ifc_flags == 0);
3800 
3801 	ifp = ctx->ifc_ifp = if_gethandle(IFT_ETHER);
3802 	if (ifp == NULL) {
3803 		device_printf(dev, "can not allocate ifnet structure\n");
3804 		return (ENOMEM);
3805 	}
3806 
3807 	/*
3808 	 * Initialize our context's device specific methods
3809 	 */
3810 	kobj_init((kobj_t) ctx, (kobj_class_t) driver);
3811 	kobj_class_compile((kobj_class_t) driver);
3812 	driver->refs++;
3813 
3814 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
3815 	if_setsoftc(ifp, ctx);
3816 	if_setdev(ifp, dev);
3817 	if_setinitfn(ifp, iflib_if_init);
3818 	if_setioctlfn(ifp, iflib_if_ioctl);
3819 	if_settransmitfn(ifp, iflib_if_transmit);
3820 	if_setqflushfn(ifp, iflib_if_qflush);
3821 	if_setgetcounterfn(ifp, iflib_if_get_counter);
3822 	if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
3823 
3824 	if_setcapabilities(ifp, 0);
3825 	if_setcapenable(ifp, 0);
3826 
3827 	ctx->ifc_vlan_attach_event =
3828 		EVENTHANDLER_REGISTER(vlan_config, iflib_vlan_register, ctx,
3829 							  EVENTHANDLER_PRI_FIRST);
3830 	ctx->ifc_vlan_detach_event =
3831 		EVENTHANDLER_REGISTER(vlan_unconfig, iflib_vlan_unregister, ctx,
3832 							  EVENTHANDLER_PRI_FIRST);
3833 
3834 	ifmedia_init(&ctx->ifc_media, IFM_IMASK,
3835 					 iflib_media_change, iflib_media_status);
3836 
3837 	return (0);
3838 }
3839 
3840 
3841 static int
3842 iflib_queues_alloc(if_ctx_t ctx)
3843 {
3844 	if_shared_ctx_t sctx = ctx->ifc_sctx;
3845 	device_t dev = ctx->ifc_dev;
3846 	int nrxqsets = ctx->ifc_softc_ctx.isc_nrxqsets;
3847 	int ntxqsets = ctx->ifc_softc_ctx.isc_ntxqsets;
3848 	iflib_txq_t txq;
3849 	iflib_rxq_t rxq;
3850 	iflib_fl_t fl = NULL;
3851 	int i, j, err, txconf, rxconf, fl_ifdi_offset;
3852 	iflib_dma_info_t ifdip;
3853 	uint32_t *rxqsizes = sctx->isc_rxqsizes;
3854 	uint32_t *txqsizes = sctx->isc_txqsizes;
3855 	uint8_t nrxqs = sctx->isc_nrxqs;
3856 	uint8_t ntxqs = sctx->isc_ntxqs;
3857 	int nfree_lists = sctx->isc_nfl ? sctx->isc_nfl : 1;
3858 	caddr_t *vaddrs;
3859 	uint64_t *paddrs;
3860 	struct ifmp_ring **brscp;
3861 	int nbuf_rings = 1; /* XXX determine dynamically */
3862 
3863 	KASSERT(ntxqs > 0, ("number of queues must be at least 1"));
3864 	KASSERT(nrxqs > 0, ("number of queues must be at least 1"));
3865 
3866 	brscp = NULL;
3867 	rxq = NULL;
3868 
3869 /* Allocate the TX ring struct memory */
3870 	if (!(txq =
3871 	    (iflib_txq_t) malloc(sizeof(struct iflib_txq) *
3872 	    ntxqsets, M_IFLIB, M_NOWAIT | M_ZERO))) {
3873 		device_printf(dev, "Unable to allocate TX ring memory\n");
3874 		err = ENOMEM;
3875 		goto fail;
3876 	}
3877 
3878 	/* Now allocate the RX */
3879 	if (!(rxq =
3880 	    (iflib_rxq_t) malloc(sizeof(struct iflib_rxq) *
3881 	    nrxqsets, M_IFLIB, M_NOWAIT | M_ZERO))) {
3882 		device_printf(dev, "Unable to allocate RX ring memory\n");
3883 		err = ENOMEM;
3884 		goto rx_fail;
3885 	}
3886 	if (!(brscp = malloc(sizeof(void *) * nbuf_rings * nrxqsets, M_IFLIB, M_NOWAIT | M_ZERO))) {
3887 		device_printf(dev, "Unable to buf_ring_sc * memory\n");
3888 		err = ENOMEM;
3889 		goto rx_fail;
3890 	}
3891 
3892 	ctx->ifc_txqs = txq;
3893 	ctx->ifc_rxqs = rxq;
3894 	txq = NULL;
3895 	rxq = NULL;
3896 
3897 	/*
3898 	 * XXX handle allocation failure
3899 	 */
3900 	for (txconf = i = 0; i < ntxqsets; i++, txconf++, txq++) {
3901 		/* Set up some basics */
3902 
3903 		if ((ifdip = malloc(sizeof(struct iflib_dma_info) * ntxqs, M_IFLIB, M_WAITOK|M_ZERO)) == NULL) {
3904 			device_printf(dev, "failed to allocate iflib_dma_info\n");
3905 			err = ENOMEM;
3906 			goto err_tx_desc;
3907 		}
3908 		txq->ift_ifdi = ifdip;
3909 		for (j = 0; j < ntxqs; j++, ifdip++) {
3910 			if (iflib_dma_alloc(ctx, txqsizes[j], ifdip, BUS_DMA_NOWAIT)) {
3911 				device_printf(dev, "Unable to allocate Descriptor memory\n");
3912 				err = ENOMEM;
3913 				goto err_tx_desc;
3914 			}
3915 			bzero((void *)ifdip->idi_vaddr, txqsizes[j]);
3916 		}
3917 		txq->ift_ctx = ctx;
3918 		txq->ift_id = i;
3919 		/* XXX fix this */
3920 		txq->ift_timer.c_cpu = i % mp_ncpus;
3921 		txq->ift_db_check.c_cpu = i % mp_ncpus;
3922 		txq->ift_nbr = nbuf_rings;
3923 
3924 		if (iflib_txsd_alloc(txq)) {
3925 			device_printf(dev, "Critical Failure setting up TX buffers\n");
3926 			err = ENOMEM;
3927 			goto err_tx_desc;
3928 		}
3929 
3930 		/* Initialize the TX lock */
3931 		snprintf(txq->ift_mtx_name, MTX_NAME_LEN, "%s:tx(%d):callout",
3932 		    device_get_nameunit(dev), txq->ift_id);
3933 		mtx_init(&txq->ift_mtx, txq->ift_mtx_name, NULL, MTX_DEF);
3934 		callout_init_mtx(&txq->ift_timer, &txq->ift_mtx, 0);
3935 		callout_init_mtx(&txq->ift_db_check, &txq->ift_mtx, 0);
3936 
3937 		snprintf(txq->ift_db_mtx_name, MTX_NAME_LEN, "%s:tx(%d):db",
3938 			 device_get_nameunit(dev), txq->ift_id);
3939 		TXDB_LOCK_INIT(txq);
3940 
3941 		txq->ift_br = brscp + i*nbuf_rings;
3942 		for (j = 0; j < nbuf_rings; j++) {
3943 			err = ifmp_ring_alloc(&txq->ift_br[j], 2048, txq, iflib_txq_drain,
3944 					      iflib_txq_can_drain, M_IFLIB, M_WAITOK);
3945 			if (err) {
3946 				/* XXX free any allocated rings */
3947 				device_printf(dev, "Unable to allocate buf_ring\n");
3948 				goto err_tx_desc;
3949 			}
3950 		}
3951 	}
3952 
3953 	for (rxconf = i = 0; i < nrxqsets; i++, rxconf++, rxq++) {
3954 		/* Set up some basics */
3955 
3956 		if ((ifdip = malloc(sizeof(struct iflib_dma_info) * nrxqs, M_IFLIB, M_WAITOK|M_ZERO)) == NULL) {
3957 			device_printf(dev, "failed to allocate iflib_dma_info\n");
3958 			err = ENOMEM;
3959 			goto err_tx_desc;
3960 		}
3961 
3962 		rxq->ifr_ifdi = ifdip;
3963 		for (j = 0; j < nrxqs; j++, ifdip++) {
3964 			if (iflib_dma_alloc(ctx, rxqsizes[j], ifdip, BUS_DMA_NOWAIT)) {
3965 				device_printf(dev, "Unable to allocate Descriptor memory\n");
3966 				err = ENOMEM;
3967 				goto err_tx_desc;
3968 			}
3969 			bzero((void *)ifdip->idi_vaddr, rxqsizes[j]);
3970 		}
3971 		rxq->ifr_ctx = ctx;
3972 		rxq->ifr_id = i;
3973 		if (sctx->isc_flags & IFLIB_HAS_CQ) {
3974 			fl_ifdi_offset = 1;
3975 		} else {
3976 			fl_ifdi_offset = 0;
3977 		}
3978 		rxq->ifr_nfl = nfree_lists;
3979 		if (!(fl =
3980 			  (iflib_fl_t) malloc(sizeof(struct iflib_fl) * nfree_lists, M_IFLIB, M_NOWAIT | M_ZERO))) {
3981 			device_printf(dev, "Unable to allocate free list memory\n");
3982 			err = ENOMEM;
3983 			goto err_tx_desc;
3984 		}
3985 		rxq->ifr_fl = fl;
3986 		for (j = 0; j < nfree_lists; j++) {
3987 			rxq->ifr_fl[j].ifl_rxq = rxq;
3988 			rxq->ifr_fl[j].ifl_id = j;
3989 			rxq->ifr_fl[j].ifl_ifdi = &rxq->ifr_ifdi[j + fl_ifdi_offset];
3990 		}
3991         /* Allocate receive buffers for the ring*/
3992 		if (iflib_rxsd_alloc(rxq)) {
3993 			device_printf(dev,
3994 			    "Critical Failure setting up receive buffers\n");
3995 			err = ENOMEM;
3996 			goto err_rx_desc;
3997 		}
3998 	}
3999 
4000 	/* TXQs */
4001 	vaddrs = malloc(sizeof(caddr_t)*ntxqsets*ntxqs, M_IFLIB, M_WAITOK);
4002 	paddrs = malloc(sizeof(uint64_t)*ntxqsets*ntxqs, M_IFLIB, M_WAITOK);
4003 	for (i = 0; i < ntxqsets; i++) {
4004 		iflib_dma_info_t di = ctx->ifc_txqs[i].ift_ifdi;
4005 
4006 		for (j = 0; j < ntxqs; j++, di++) {
4007 			vaddrs[i*ntxqs + j] = di->idi_vaddr;
4008 			paddrs[i*ntxqs + j] = di->idi_paddr;
4009 		}
4010 	}
4011 	if ((err = IFDI_TX_QUEUES_ALLOC(ctx, vaddrs, paddrs, ntxqs, ntxqsets)) != 0) {
4012 		device_printf(ctx->ifc_dev, "device queue allocation failed\n");
4013 		iflib_tx_structures_free(ctx);
4014 		free(vaddrs, M_IFLIB);
4015 		free(paddrs, M_IFLIB);
4016 		goto err_rx_desc;
4017 	}
4018 	free(vaddrs, M_IFLIB);
4019 	free(paddrs, M_IFLIB);
4020 
4021 	/* RXQs */
4022 	vaddrs = malloc(sizeof(caddr_t)*nrxqsets*nrxqs, M_IFLIB, M_WAITOK);
4023 	paddrs = malloc(sizeof(uint64_t)*nrxqsets*nrxqs, M_IFLIB, M_WAITOK);
4024 	for (i = 0; i < nrxqsets; i++) {
4025 		iflib_dma_info_t di = ctx->ifc_rxqs[i].ifr_ifdi;
4026 
4027 		for (j = 0; j < nrxqs; j++, di++) {
4028 			vaddrs[i*nrxqs + j] = di->idi_vaddr;
4029 			paddrs[i*nrxqs + j] = di->idi_paddr;
4030 		}
4031 	}
4032 	if ((err = IFDI_RX_QUEUES_ALLOC(ctx, vaddrs, paddrs, nrxqs, nrxqsets)) != 0) {
4033 		device_printf(ctx->ifc_dev, "device queue allocation failed\n");
4034 		iflib_tx_structures_free(ctx);
4035 		free(vaddrs, M_IFLIB);
4036 		free(paddrs, M_IFLIB);
4037 		goto err_rx_desc;
4038 	}
4039 	free(vaddrs, M_IFLIB);
4040 	free(paddrs, M_IFLIB);
4041 
4042 	return (0);
4043 
4044 /* XXX handle allocation failure changes */
4045 err_rx_desc:
4046 err_tx_desc:
4047 	if (ctx->ifc_rxqs != NULL)
4048 		free(ctx->ifc_rxqs, M_IFLIB);
4049 	ctx->ifc_rxqs = NULL;
4050 	if (ctx->ifc_txqs != NULL)
4051 		free(ctx->ifc_txqs, M_IFLIB);
4052 	ctx->ifc_txqs = NULL;
4053 rx_fail:
4054 	if (brscp != NULL)
4055 		free(brscp, M_IFLIB);
4056 	if (rxq != NULL)
4057 		free(rxq, M_IFLIB);
4058 	if (txq != NULL)
4059 		free(txq, M_IFLIB);
4060 fail:
4061 	return (err);
4062 }
4063 
4064 static int
4065 iflib_tx_structures_setup(if_ctx_t ctx)
4066 {
4067 	iflib_txq_t txq = ctx->ifc_txqs;
4068 	int i;
4069 
4070 	for (i = 0; i < NTXQSETS(ctx); i++, txq++)
4071 		iflib_txq_setup(txq);
4072 
4073 	return (0);
4074 }
4075 
4076 static void
4077 iflib_tx_structures_free(if_ctx_t ctx)
4078 {
4079 	iflib_txq_t txq = ctx->ifc_txqs;
4080 	int i, j;
4081 
4082 	for (i = 0; i < NTXQSETS(ctx); i++, txq++) {
4083 		iflib_txq_destroy(txq);
4084 		for (j = 0; j < ctx->ifc_nhwtxqs; j++)
4085 			iflib_dma_free(&txq->ift_ifdi[j]);
4086 	}
4087 	free(ctx->ifc_txqs, M_IFLIB);
4088 	ctx->ifc_txqs = NULL;
4089 	IFDI_QUEUES_FREE(ctx);
4090 }
4091 
4092 /*********************************************************************
4093  *
4094  *  Initialize all receive rings.
4095  *
4096  **********************************************************************/
4097 static int
4098 iflib_rx_structures_setup(if_ctx_t ctx)
4099 {
4100 	iflib_rxq_t rxq = ctx->ifc_rxqs;
4101 	int q;
4102 #if defined(INET6) || defined(INET)
4103 	int i, err;
4104 #endif
4105 
4106 	for (q = 0; q < ctx->ifc_softc_ctx.isc_nrxqsets; q++, rxq++) {
4107 #if defined(INET6) || defined(INET)
4108 		tcp_lro_free(&rxq->ifr_lc);
4109 		if ((err = tcp_lro_init(&rxq->ifr_lc)) != 0) {
4110 			device_printf(ctx->ifc_dev, "LRO Initialization failed!\n");
4111 			goto fail;
4112 		}
4113 		rxq->ifr_lro_enabled = TRUE;
4114 		rxq->ifr_lc.ifp = ctx->ifc_ifp;
4115 #endif
4116 		IFDI_RXQ_SETUP(ctx, rxq->ifr_id);
4117 	}
4118 	return (0);
4119 #if defined(INET6) || defined(INET)
4120 fail:
4121 	/*
4122 	 * Free RX software descriptors allocated so far, we will only handle
4123 	 * the rings that completed, the failing case will have
4124 	 * cleaned up for itself. 'q' failed, so its the terminus.
4125 	 */
4126 	rxq = ctx->ifc_rxqs;
4127 	for (i = 0; i < q; ++i, rxq++) {
4128 		iflib_rx_sds_free(rxq);
4129 		rxq->ifr_cq_gen = rxq->ifr_cq_cidx = rxq->ifr_cq_pidx = 0;
4130 	}
4131 	return (err);
4132 #endif
4133 }
4134 
4135 /*********************************************************************
4136  *
4137  *  Free all receive rings.
4138  *
4139  **********************************************************************/
4140 static void
4141 iflib_rx_structures_free(if_ctx_t ctx)
4142 {
4143 	iflib_rxq_t rxq = ctx->ifc_rxqs;
4144 
4145 	for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxqsets; i++, rxq++) {
4146 		iflib_rx_sds_free(rxq);
4147 	}
4148 }
4149 
4150 static int
4151 iflib_qset_structures_setup(if_ctx_t ctx)
4152 {
4153 	int err;
4154 
4155 	if ((err = iflib_tx_structures_setup(ctx)) != 0)
4156 		return (err);
4157 
4158 	if ((err = iflib_rx_structures_setup(ctx)) != 0) {
4159 		device_printf(ctx->ifc_dev, "iflib_rx_structures_setup failed: %d\n", err);
4160 		iflib_tx_structures_free(ctx);
4161 		iflib_rx_structures_free(ctx);
4162 	}
4163 	return (err);
4164 }
4165 
4166 int
4167 iflib_irq_alloc(if_ctx_t ctx, if_irq_t irq, int rid,
4168 				driver_filter_t filter, void *filter_arg, driver_intr_t handler, void *arg, char *name)
4169 {
4170 
4171 	return (_iflib_irq_alloc(ctx, irq, rid, filter, handler, arg, name));
4172 }
4173 
4174 static void
4175 find_nth(if_ctx_t ctx, cpuset_t *cpus, int qid)
4176 {
4177 	int i, cpuid;
4178 
4179 	CPU_COPY(&ctx->ifc_cpus, cpus);
4180 	/* clear up to the qid'th bit */
4181 	for (i = 0; i < qid; i++) {
4182 		cpuid = CPU_FFS(cpus);
4183 		CPU_CLR(cpuid, cpus);
4184 	}
4185 }
4186 
4187 int
4188 iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid,
4189 						iflib_intr_type_t type, driver_filter_t *filter,
4190 						void *filter_arg, int qid, char *name)
4191 {
4192 	struct grouptask *gtask;
4193 	struct taskqgroup *tqg;
4194 	iflib_filter_info_t info;
4195 	cpuset_t cpus;
4196 	task_fn_t *fn;
4197 	int tqrid, err;
4198 	void *q;
4199 
4200 	info = &ctx->ifc_filter_info;
4201 
4202 	switch (type) {
4203 	/* XXX merge tx/rx for netmap? */
4204 	case IFLIB_INTR_TX:
4205 		q = &ctx->ifc_txqs[qid];
4206 		info = &ctx->ifc_txqs[qid].ift_filter_info;
4207 		gtask = &ctx->ifc_txqs[qid].ift_task;
4208 		tqg = qgroup_if_io_tqg;
4209 		tqrid = irq->ii_rid;
4210 		fn = _task_fn_tx;
4211 		break;
4212 	case IFLIB_INTR_RX:
4213 		q = &ctx->ifc_rxqs[qid];
4214 		info = &ctx->ifc_rxqs[qid].ifr_filter_info;
4215 		gtask = &ctx->ifc_rxqs[qid].ifr_task;
4216 		tqg = qgroup_if_io_tqg;
4217 		tqrid = irq->ii_rid;
4218 		fn = _task_fn_rx;
4219 		break;
4220 	case IFLIB_INTR_ADMIN:
4221 		q = ctx;
4222 		info = &ctx->ifc_filter_info;
4223 		gtask = &ctx->ifc_admin_task;
4224 		tqg = qgroup_if_config_tqg;
4225 		tqrid = -1;
4226 		fn = _task_fn_admin;
4227 		break;
4228 	default:
4229 		panic("unknown net intr type");
4230 	}
4231 	GROUPTASK_INIT(gtask, 0, fn, q);
4232 
4233 	info->ifi_filter = filter;
4234 	info->ifi_filter_arg = filter_arg;
4235 	info->ifi_task = gtask;
4236 
4237 	/* XXX query cpu that rid belongs to */
4238 
4239 	err = _iflib_irq_alloc(ctx, irq, rid, iflib_fast_intr, NULL, info,  name);
4240 	if (err != 0)
4241 		return (err);
4242 	if (tqrid != -1) {
4243 		find_nth(ctx, &cpus, qid);
4244 		taskqgroup_attach_cpu(tqg, gtask, q, CPU_FFS(&cpus), irq->ii_rid, name);
4245 	} else
4246 		taskqgroup_attach(tqg, gtask, q, tqrid, name);
4247 
4248 
4249 	return (0);
4250 }
4251 
4252 void
4253 iflib_softirq_alloc_generic(if_ctx_t ctx, int rid, iflib_intr_type_t type,  void *arg, int qid, char *name)
4254 {
4255 	struct grouptask *gtask;
4256 	struct taskqgroup *tqg;
4257 	task_fn_t *fn;
4258 	void *q;
4259 
4260 	switch (type) {
4261 	case IFLIB_INTR_TX:
4262 		q = &ctx->ifc_txqs[qid];
4263 		gtask = &ctx->ifc_txqs[qid].ift_task;
4264 		tqg = qgroup_if_io_tqg;
4265 		fn = _task_fn_tx;
4266 		break;
4267 	case IFLIB_INTR_RX:
4268 		q = &ctx->ifc_rxqs[qid];
4269 		gtask = &ctx->ifc_rxqs[qid].ifr_task;
4270 		tqg = qgroup_if_io_tqg;
4271 		fn = _task_fn_rx;
4272 		break;
4273 	case IFLIB_INTR_ADMIN:
4274 		q = ctx;
4275 		gtask = &ctx->ifc_admin_task;
4276 		tqg = qgroup_if_config_tqg;
4277 		rid = -1;
4278 		fn = _task_fn_admin;
4279 		break;
4280 	case IFLIB_INTR_IOV:
4281 		q = ctx;
4282 		gtask = &ctx->ifc_vflr_task;
4283 		tqg = qgroup_if_config_tqg;
4284 		rid = -1;
4285 		fn = _task_fn_iov;
4286 		break;
4287 	default:
4288 		panic("unknown net intr type");
4289 	}
4290 	GROUPTASK_INIT(gtask, 0, fn, q);
4291 	taskqgroup_attach(tqg, gtask, q, rid, name);
4292 }
4293 
4294 void
4295 iflib_irq_free(if_ctx_t ctx, if_irq_t irq)
4296 {
4297 	if (irq->ii_tag)
4298 		bus_teardown_intr(ctx->ifc_dev, irq->ii_res, irq->ii_tag);
4299 
4300 	if (irq->ii_res)
4301 		bus_release_resource(ctx->ifc_dev, SYS_RES_IRQ, irq->ii_rid, irq->ii_res);
4302 }
4303 
4304 static int
4305 iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filter_arg, int *rid, char *name)
4306 {
4307 	iflib_txq_t txq = ctx->ifc_txqs;
4308 	iflib_rxq_t rxq = ctx->ifc_rxqs;
4309 	if_irq_t irq = &ctx->ifc_legacy_irq;
4310 	iflib_filter_info_t info;
4311 	struct grouptask *gtask;
4312 	struct taskqgroup *tqg;
4313 	task_fn_t *fn;
4314 	int tqrid;
4315 	void *q;
4316 	int err;
4317 
4318 	q = &ctx->ifc_rxqs[0];
4319 	info = &rxq[0].ifr_filter_info;
4320 	gtask = &rxq[0].ifr_task;
4321 	tqg = qgroup_if_io_tqg;
4322 	tqrid = irq->ii_rid = *rid;
4323 	fn = _task_fn_rx;
4324 
4325 	ctx->ifc_flags |= IFC_LEGACY;
4326 	info->ifi_filter = filter;
4327 	info->ifi_filter_arg = filter_arg;
4328 	info->ifi_task = gtask;
4329 
4330 	/* We allocate a single interrupt resource */
4331 	if ((err = _iflib_irq_alloc(ctx, irq, tqrid, iflib_fast_intr, NULL, info, name)) != 0)
4332 		return (err);
4333 	GROUPTASK_INIT(gtask, 0, fn, q);
4334 	taskqgroup_attach(tqg, gtask, q, tqrid, name);
4335 
4336 	GROUPTASK_INIT(&txq->ift_task, 0, _task_fn_tx, txq);
4337 	taskqgroup_attach(qgroup_if_io_tqg, &txq->ift_task, txq, tqrid, "tx");
4338 	GROUPTASK_INIT(&ctx->ifc_admin_task, 0, _task_fn_admin, ctx);
4339 	taskqgroup_attach(qgroup_if_config_tqg, &ctx->ifc_admin_task, ctx, -1, "admin/link");
4340 
4341 	return (0);
4342 }
4343 
4344 void
4345 iflib_led_create(if_ctx_t ctx)
4346 {
4347 
4348 	ctx->ifc_led_dev = led_create(iflib_led_func, ctx,
4349 								  device_get_nameunit(ctx->ifc_dev));
4350 }
4351 
4352 void
4353 iflib_tx_intr_deferred(if_ctx_t ctx, int txqid)
4354 {
4355 
4356 	GROUPTASK_ENQUEUE(&ctx->ifc_txqs[txqid].ift_task);
4357 }
4358 
4359 void
4360 iflib_rx_intr_deferred(if_ctx_t ctx, int rxqid)
4361 {
4362 
4363 	GROUPTASK_ENQUEUE(&ctx->ifc_rxqs[rxqid].ifr_task);
4364 }
4365 
4366 void
4367 iflib_admin_intr_deferred(if_ctx_t ctx)
4368 {
4369 
4370 	GROUPTASK_ENQUEUE(&ctx->ifc_admin_task);
4371 }
4372 
4373 void
4374 iflib_iov_intr_deferred(if_ctx_t ctx)
4375 {
4376 
4377 	GROUPTASK_ENQUEUE(&ctx->ifc_vflr_task);
4378 }
4379 
4380 void
4381 iflib_io_tqg_attach(struct grouptask *gt, void *uniq, int cpu, char *name)
4382 {
4383 
4384 	taskqgroup_attach_cpu(qgroup_if_io_tqg, gt, uniq, cpu, -1, name);
4385 }
4386 
4387 void
4388 iflib_config_gtask_init(if_ctx_t ctx, struct grouptask *gtask, task_fn_t *fn,
4389 	char *name)
4390 {
4391 
4392 	GROUPTASK_INIT(gtask, 0, fn, ctx);
4393 	taskqgroup_attach(qgroup_if_config_tqg, gtask, gtask, -1, name);
4394 }
4395 
4396 void
4397 iflib_link_state_change(if_ctx_t ctx, int link_state)
4398 {
4399 	if_t ifp = ctx->ifc_ifp;
4400 	iflib_txq_t txq = ctx->ifc_txqs;
4401 
4402 #if 0
4403 	if_setbaudrate(ifp, baudrate);
4404 #endif
4405 	/* If link down, disable watchdog */
4406 	if ((ctx->ifc_link_state == LINK_STATE_UP) && (link_state == LINK_STATE_DOWN)) {
4407 		for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxqsets; i++, txq++)
4408 			txq->ift_qstatus = IFLIB_QUEUE_IDLE;
4409 	}
4410 	ctx->ifc_link_state = link_state;
4411 	if_link_state_change(ifp, link_state);
4412 }
4413 
4414 static int
4415 iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq)
4416 {
4417 	int credits;
4418 
4419 	if (ctx->isc_txd_credits_update == NULL)
4420 		return (0);
4421 
4422 	if ((credits = ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, txq->ift_cidx_processed, true)) == 0)
4423 		return (0);
4424 
4425 	txq->ift_processed += credits;
4426 	txq->ift_cidx_processed += credits;
4427 
4428 	if (txq->ift_cidx_processed >= txq->ift_size)
4429 		txq->ift_cidx_processed -= txq->ift_size;
4430 	return (credits);
4431 }
4432 
4433 static int
4434 iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, int cidx)
4435 {
4436 
4437 	return (ctx->isc_rxd_available(ctx->ifc_softc, rxq->ifr_id, cidx));
4438 }
4439 
4440 void
4441 iflib_add_int_delay_sysctl(if_ctx_t ctx, const char *name,
4442 	const char *description, if_int_delay_info_t info,
4443 	int offset, int value)
4444 {
4445 	info->iidi_ctx = ctx;
4446 	info->iidi_offset = offset;
4447 	info->iidi_value = value;
4448 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(ctx->ifc_dev),
4449 	    SYSCTL_CHILDREN(device_get_sysctl_tree(ctx->ifc_dev)),
4450 	    OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
4451 	    info, 0, iflib_sysctl_int_delay, "I", description);
4452 }
4453 
4454 struct mtx *
4455 iflib_ctx_lock_get(if_ctx_t ctx)
4456 {
4457 
4458 	return (&ctx->ifc_mtx);
4459 }
4460 
4461 static int
4462 iflib_msix_init(if_ctx_t ctx)
4463 {
4464 	device_t dev = ctx->ifc_dev;
4465 	if_shared_ctx_t sctx = ctx->ifc_sctx;
4466 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
4467 	int vectors, queues, rx_queues, tx_queues, queuemsgs, msgs;
4468 	int iflib_num_tx_queues, iflib_num_rx_queues;
4469 	int err, admincnt, bar;
4470 
4471 	iflib_num_tx_queues = ctx->ifc_sysctl_ntxqs;
4472 	iflib_num_rx_queues = ctx->ifc_sysctl_nrxqs;
4473 	bar = ctx->ifc_softc_ctx.isc_msix_bar;
4474 	admincnt = sctx->isc_admin_intrcnt;
4475 	/* Override by tuneable */
4476 	if (enable_msix == 0)
4477 		goto msi;
4478 
4479 	/*
4480 	** When used in a virtualized environment
4481 	** PCI BUSMASTER capability may not be set
4482 	** so explicity set it here and rewrite
4483 	** the ENABLE in the MSIX control register
4484 	** at this point to cause the host to
4485 	** successfully initialize us.
4486 	*/
4487 	{
4488 		uint16_t pci_cmd_word;
4489 		int msix_ctrl, rid;
4490 
4491 		rid = 0;
4492 		pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
4493 		pci_cmd_word |= PCIM_CMD_BUSMASTEREN;
4494 		pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2);
4495 		pci_find_cap(dev, PCIY_MSIX, &rid);
4496 		rid += PCIR_MSIX_CTRL;
4497 		msix_ctrl = pci_read_config(dev, rid, 2);
4498 		msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
4499 		pci_write_config(dev, rid, msix_ctrl, 2);
4500 	}
4501 
4502 	/*
4503 	 * bar == -1 => "trust me I know what I'm doing"
4504 	 * https://www.youtube.com/watch?v=nnwWKkNau4I
4505 	 * Some drivers are for hardware that is so shoddily
4506 	 * documented that no one knows which bars are which
4507 	 * so the developer has to map all bars. This hack
4508 	 * allows shoddy garbage to use msix in this framework.
4509 	 */
4510 	if (bar != -1) {
4511 		ctx->ifc_msix_mem = bus_alloc_resource_any(dev,
4512 	            SYS_RES_MEMORY, &bar, RF_ACTIVE);
4513 		if (ctx->ifc_msix_mem == NULL) {
4514 			/* May not be enabled */
4515 			device_printf(dev, "Unable to map MSIX table \n");
4516 			goto msi;
4517 		}
4518 	}
4519 	/* First try MSI/X */
4520 	if ((msgs = pci_msix_count(dev)) == 0) { /* system has msix disabled */
4521 		device_printf(dev, "System has MSIX disabled \n");
4522 		bus_release_resource(dev, SYS_RES_MEMORY,
4523 		    bar, ctx->ifc_msix_mem);
4524 		ctx->ifc_msix_mem = NULL;
4525 		goto msi;
4526 	}
4527 #if IFLIB_DEBUG
4528 	/* use only 1 qset in debug mode */
4529 	queuemsgs = min(msgs - admincnt, 1);
4530 #else
4531 	queuemsgs = msgs - admincnt;
4532 #endif
4533 	if (bus_get_cpus(dev, INTR_CPUS, sizeof(ctx->ifc_cpus), &ctx->ifc_cpus) == 0) {
4534 #ifdef RSS
4535 		queues = imin(queuemsgs, rss_getnumbuckets());
4536 #else
4537 		queues = queuemsgs;
4538 #endif
4539 		queues = imin(CPU_COUNT(&ctx->ifc_cpus), queues);
4540 		device_printf(dev, "pxm cpus: %d queue msgs: %d admincnt: %d\n",
4541 					  CPU_COUNT(&ctx->ifc_cpus), queuemsgs, admincnt);
4542 	} else {
4543 		device_printf(dev, "Unable to fetch CPU list\n");
4544 		/* Figure out a reasonable auto config value */
4545 		queues = min(queuemsgs, mp_ncpus);
4546 	}
4547 #ifdef  RSS
4548 	/* If we're doing RSS, clamp at the number of RSS buckets */
4549 	if (queues > rss_getnumbuckets())
4550 		queues = rss_getnumbuckets();
4551 #endif
4552 	if (iflib_num_rx_queues > 0 && iflib_num_rx_queues < queues)
4553 		queues = rx_queues = iflib_num_rx_queues;
4554 	else
4555 		rx_queues = queues;
4556 	if (iflib_num_tx_queues > 0 && iflib_num_tx_queues < queues)
4557 		tx_queues = iflib_num_tx_queues;
4558 	else
4559 		tx_queues = queues;
4560 
4561 	device_printf(dev, "using %d rx queues %d tx queues \n", rx_queues, tx_queues);
4562 
4563 	vectors = queues + admincnt;
4564 	if ((err = pci_alloc_msix(dev, &vectors)) == 0) {
4565 		device_printf(dev,
4566 					  "Using MSIX interrupts with %d vectors\n", vectors);
4567 		scctx->isc_vectors = vectors;
4568 		scctx->isc_nrxqsets = rx_queues;
4569 		scctx->isc_ntxqsets = tx_queues;
4570 		scctx->isc_intr = IFLIB_INTR_MSIX;
4571 		return (vectors);
4572 	} else {
4573 		device_printf(dev, "failed to allocate %d msix vectors, err: %d - using MSI\n", vectors, err);
4574 	}
4575 msi:
4576 	vectors = pci_msi_count(dev);
4577 	scctx->isc_nrxqsets = 1;
4578 	scctx->isc_ntxqsets = 1;
4579 	scctx->isc_vectors = vectors;
4580 	if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0) {
4581 		device_printf(dev,"Using an MSI interrupt\n");
4582 		scctx->isc_intr = IFLIB_INTR_MSI;
4583 	} else {
4584 		device_printf(dev,"Using a Legacy interrupt\n");
4585 		scctx->isc_intr = IFLIB_INTR_LEGACY;
4586 	}
4587 
4588 	return (vectors);
4589 }
4590 
4591 char * ring_states[] = { "IDLE", "BUSY", "STALLED", "ABDICATED" };
4592 
4593 static int
4594 mp_ring_state_handler(SYSCTL_HANDLER_ARGS)
4595 {
4596 	int rc;
4597 	uint16_t *state = ((uint16_t *)oidp->oid_arg1);
4598 	struct sbuf *sb;
4599 	char *ring_state = "UNKNOWN";
4600 
4601 	/* XXX needed ? */
4602 	rc = sysctl_wire_old_buffer(req, 0);
4603 	MPASS(rc == 0);
4604 	if (rc != 0)
4605 		return (rc);
4606 	sb = sbuf_new_for_sysctl(NULL, NULL, 80, req);
4607 	MPASS(sb != NULL);
4608 	if (sb == NULL)
4609 		return (ENOMEM);
4610 	if (state[3] <= 3)
4611 		ring_state = ring_states[state[3]];
4612 
4613 	sbuf_printf(sb, "pidx_head: %04hd pidx_tail: %04hd cidx: %04hd state: %s",
4614 		    state[0], state[1], state[2], ring_state);
4615 	rc = sbuf_finish(sb);
4616 	sbuf_delete(sb);
4617         return(rc);
4618 }
4619 
4620 
4621 
4622 #define NAME_BUFLEN 32
4623 static void
4624 iflib_add_device_sysctl_pre(if_ctx_t ctx)
4625 {
4626         device_t dev = iflib_get_dev(ctx);
4627 	struct sysctl_oid_list *child, *oid_list;
4628 	struct sysctl_ctx_list *ctx_list;
4629 	struct sysctl_oid *node;
4630 
4631 	ctx_list = device_get_sysctl_ctx(dev);
4632 	child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
4633 	ctx->ifc_sysctl_node = node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, "iflib",
4634 						      CTLFLAG_RD, NULL, "IFLIB fields");
4635 	oid_list = SYSCTL_CHILDREN(node);
4636 
4637 	SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_ntxqs",
4638 		       CTLFLAG_RWTUN, &ctx->ifc_sysctl_ntxqs, 0,
4639 			"# of txqs to use, 0 => use default #");
4640 	SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_nrxqs",
4641 		       CTLFLAG_RWTUN, &ctx->ifc_sysctl_ntxqs, 0,
4642 			"# of txqs to use, 0 => use default #");
4643 	SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_ntxds",
4644 		       CTLFLAG_RWTUN, &ctx->ifc_sysctl_ntxds, 0,
4645 			"# of tx descriptors to use, 0 => use default #");
4646 	SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_nrxds",
4647 		       CTLFLAG_RWTUN, &ctx->ifc_sysctl_nrxds, 0,
4648 			"# of rx descriptors to use, 0 => use default #");
4649 
4650 }
4651 
4652 static void
4653 iflib_add_device_sysctl_post(if_ctx_t ctx)
4654 {
4655 	if_shared_ctx_t sctx = ctx->ifc_sctx;
4656 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
4657         device_t dev = iflib_get_dev(ctx);
4658 	struct sysctl_oid_list *child;
4659 	struct sysctl_ctx_list *ctx_list;
4660 	iflib_fl_t fl;
4661 	iflib_txq_t txq;
4662 	iflib_rxq_t rxq;
4663 	int i, j;
4664 	char namebuf[NAME_BUFLEN];
4665 	char *qfmt;
4666 	struct sysctl_oid *queue_node, *fl_node, *node;
4667 	struct sysctl_oid_list *queue_list, *fl_list;
4668 	ctx_list = device_get_sysctl_ctx(dev);
4669 
4670 	node = ctx->ifc_sysctl_node;
4671 	child = SYSCTL_CHILDREN(node);
4672 
4673 	if (scctx->isc_ntxqsets > 100)
4674 		qfmt = "txq%03d";
4675 	else if (scctx->isc_ntxqsets > 10)
4676 		qfmt = "txq%02d";
4677 	else
4678 		qfmt = "txq%d";
4679 	for (i = 0, txq = ctx->ifc_txqs; i < scctx->isc_ntxqsets; i++, txq++) {
4680 		snprintf(namebuf, NAME_BUFLEN, qfmt, i);
4681 		queue_node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, namebuf,
4682 					     CTLFLAG_RD, NULL, "Queue Name");
4683 		queue_list = SYSCTL_CHILDREN(queue_node);
4684 #if MEMORY_LOGGING
4685 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_dequeued",
4686 				CTLFLAG_RD,
4687 				&txq->ift_dequeued, "total mbufs freed");
4688 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_enqueued",
4689 				CTLFLAG_RD,
4690 				&txq->ift_enqueued, "total mbufs enqueued");
4691 #endif
4692 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "mbuf_defrag",
4693 				   CTLFLAG_RD,
4694 				   &txq->ift_mbuf_defrag, "# of times m_defrag was called");
4695 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "m_pullups",
4696 				   CTLFLAG_RD,
4697 				   &txq->ift_pullups, "# of times m_pullup was called");
4698 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "mbuf_defrag_failed",
4699 				   CTLFLAG_RD,
4700 				   &txq->ift_mbuf_defrag_failed, "# of times m_defrag failed");
4701 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "no_desc_avail",
4702 				   CTLFLAG_RD,
4703 				   &txq->ift_mbuf_defrag_failed, "# of times no descriptors were available");
4704 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "tx_map_failed",
4705 				   CTLFLAG_RD,
4706 				   &txq->ift_map_failed, "# of times dma map failed");
4707 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txd_encap_efbig",
4708 				   CTLFLAG_RD,
4709 				   &txq->ift_txd_encap_efbig, "# of times txd_encap returned EFBIG");
4710 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "no_tx_dma_setup",
4711 				   CTLFLAG_RD,
4712 				   &txq->ift_no_tx_dma_setup, "# of times map failed for other than EFBIG");
4713 		SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "txq_pidx",
4714 				   CTLFLAG_RD,
4715 				   &txq->ift_pidx, 1, "Producer Index");
4716 		SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "txq_cidx",
4717 				   CTLFLAG_RD,
4718 				   &txq->ift_cidx, 1, "Consumer Index");
4719 		SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "txq_cidx_processed",
4720 				   CTLFLAG_RD,
4721 				   &txq->ift_cidx_processed, 1, "Consumer Index seen by credit update");
4722 		SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "txq_in_use",
4723 				   CTLFLAG_RD,
4724 				   &txq->ift_in_use, 1, "descriptors in use");
4725 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_processed",
4726 				   CTLFLAG_RD,
4727 				   &txq->ift_processed, "descriptors procesed for clean");
4728 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_cleaned",
4729 				   CTLFLAG_RD,
4730 				   &txq->ift_cleaned, "total cleaned");
4731 		SYSCTL_ADD_PROC(ctx_list, queue_list, OID_AUTO, "ring_state",
4732 				CTLTYPE_STRING | CTLFLAG_RD, __DEVOLATILE(uint64_t *, &txq->ift_br[0]->state),
4733 				0, mp_ring_state_handler, "A", "soft ring state");
4734 		SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_enqueues",
4735 				       CTLFLAG_RD, &txq->ift_br[0]->enqueues,
4736 				       "# of enqueues to the mp_ring for this queue");
4737 		SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_drops",
4738 				       CTLFLAG_RD, &txq->ift_br[0]->drops,
4739 				       "# of drops in the mp_ring for this queue");
4740 		SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_starts",
4741 				       CTLFLAG_RD, &txq->ift_br[0]->starts,
4742 				       "# of normal consumer starts in the mp_ring for this queue");
4743 		SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_stalls",
4744 				       CTLFLAG_RD, &txq->ift_br[0]->stalls,
4745 					       "# of consumer stalls in the mp_ring for this queue");
4746 		SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_restarts",
4747 			       CTLFLAG_RD, &txq->ift_br[0]->restarts,
4748 				       "# of consumer restarts in the mp_ring for this queue");
4749 		SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_abdications",
4750 				       CTLFLAG_RD, &txq->ift_br[0]->abdications,
4751 				       "# of consumer abdications in the mp_ring for this queue");
4752 
4753 	}
4754 
4755 	if (scctx->isc_nrxqsets > 100)
4756 		qfmt = "rxq%03d";
4757 	else if (scctx->isc_nrxqsets > 10)
4758 		qfmt = "rxq%02d";
4759 	else
4760 		qfmt = "rxq%d";
4761 	for (i = 0, rxq = ctx->ifc_rxqs; i < scctx->isc_nrxqsets; i++, rxq++) {
4762 		snprintf(namebuf, NAME_BUFLEN, qfmt, i);
4763 		queue_node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, namebuf,
4764 					     CTLFLAG_RD, NULL, "Queue Name");
4765 		queue_list = SYSCTL_CHILDREN(queue_node);
4766 		if (sctx->isc_flags & IFLIB_HAS_CQ) {
4767 			SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "rxq_cq_pidx",
4768 				       CTLFLAG_RD,
4769 				       &rxq->ifr_cq_pidx, 1, "Producer Index");
4770 			SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "rxq_cq_cidx",
4771 				       CTLFLAG_RD,
4772 				       &rxq->ifr_cq_cidx, 1, "Consumer Index");
4773 		}
4774 		for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) {
4775 			snprintf(namebuf, NAME_BUFLEN, "rxq_fl%d", j);
4776 			fl_node = SYSCTL_ADD_NODE(ctx_list, queue_list, OID_AUTO, namebuf,
4777 						     CTLFLAG_RD, NULL, "freelist Name");
4778 			fl_list = SYSCTL_CHILDREN(fl_node);
4779 			SYSCTL_ADD_U16(ctx_list, fl_list, OID_AUTO, "pidx",
4780 				       CTLFLAG_RD,
4781 				       &fl->ifl_pidx, 1, "Producer Index");
4782 			SYSCTL_ADD_U16(ctx_list, fl_list, OID_AUTO, "cidx",
4783 				       CTLFLAG_RD,
4784 				       &fl->ifl_cidx, 1, "Consumer Index");
4785 			SYSCTL_ADD_U16(ctx_list, fl_list, OID_AUTO, "credits",
4786 				       CTLFLAG_RD,
4787 				       &fl->ifl_credits, 1, "credits available");
4788 #if MEMORY_LOGGING
4789 			SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_m_enqueued",
4790 					CTLFLAG_RD,
4791 					&fl->ifl_m_enqueued, "mbufs allocated");
4792 			SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_m_dequeued",
4793 					CTLFLAG_RD,
4794 					&fl->ifl_m_dequeued, "mbufs freed");
4795 			SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_cl_enqueued",
4796 					CTLFLAG_RD,
4797 					&fl->ifl_cl_enqueued, "clusters allocated");
4798 			SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_cl_dequeued",
4799 					CTLFLAG_RD,
4800 					&fl->ifl_cl_dequeued, "clusters freed");
4801 #endif
4802 
4803 		}
4804 	}
4805 
4806 }
4807