xref: /freebsd/sys/net/iflib.c (revision 11dd9ed6647d821e7b43d4f8e64412a2623fbab5)
1 /*-
2  * Copyright (c) 2014-2017, Matthew Macy <mmacy@nextbsd.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are met:
7  *
8  *  1. Redistributions of source code must retain the above copyright notice,
9  *     this list of conditions and the following disclaimer.
10  *
11  *  2. Neither the name of Matthew Macy nor the names of its
12  *     contributors may be used to endorse or promote products derived from
13  *     this software without specific prior written permission.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
19  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
20  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
21  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
22  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
24  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
25  * POSSIBILITY OF SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30 
31 #include "opt_inet.h"
32 #include "opt_inet6.h"
33 #include "opt_acpi.h"
34 
35 #include <sys/param.h>
36 #include <sys/types.h>
37 #include <sys/bus.h>
38 #include <sys/eventhandler.h>
39 #include <sys/sockio.h>
40 #include <sys/kernel.h>
41 #include <sys/lock.h>
42 #include <sys/mutex.h>
43 #include <sys/module.h>
44 #include <sys/kobj.h>
45 #include <sys/rman.h>
46 #include <sys/sbuf.h>
47 #include <sys/smp.h>
48 #include <sys/socket.h>
49 #include <sys/sysctl.h>
50 #include <sys/syslog.h>
51 #include <sys/taskqueue.h>
52 #include <sys/limits.h>
53 
54 
55 #include <net/if.h>
56 #include <net/if_var.h>
57 #include <net/if_types.h>
58 #include <net/if_media.h>
59 #include <net/bpf.h>
60 #include <net/ethernet.h>
61 #include <net/mp_ring.h>
62 
63 #include <netinet/in.h>
64 #include <netinet/in_pcb.h>
65 #include <netinet/tcp_lro.h>
66 #include <netinet/in_systm.h>
67 #include <netinet/if_ether.h>
68 #include <netinet/ip.h>
69 #include <netinet/ip6.h>
70 #include <netinet/tcp.h>
71 
72 #include <machine/bus.h>
73 #include <machine/in_cksum.h>
74 
75 #include <vm/vm.h>
76 #include <vm/pmap.h>
77 
78 #include <dev/led/led.h>
79 #include <dev/pci/pcireg.h>
80 #include <dev/pci/pcivar.h>
81 #include <dev/pci/pci_private.h>
82 
83 #include <net/iflib.h>
84 
85 #include "ifdi_if.h"
86 
87 #if defined(__i386__) || defined(__amd64__)
88 #include <sys/memdesc.h>
89 #include <machine/bus.h>
90 #include <machine/md_var.h>
91 #include <machine/specialreg.h>
92 #include <x86/include/busdma_impl.h>
93 #include <x86/iommu/busdma_dmar.h>
94 #endif
95 
96 /*
97  * enable accounting of every mbuf as it comes in to and goes out of iflib's software descriptor references
98  */
99 #define MEMORY_LOGGING 0
100 /*
101  * Enable mbuf vectors for compressing long mbuf chains
102  */
103 
104 /*
105  * NB:
106  * - Prefetching in tx cleaning should perhaps be a tunable. The distance ahead
107  *   we prefetch needs to be determined by the time spent in m_free vis a vis
108  *   the cost of a prefetch. This will of course vary based on the workload:
109  *      - NFLX's m_free path is dominated by vm-based M_EXT manipulation which
110  *        is quite expensive, thus suggesting very little prefetch.
111  *      - small packet forwarding which is just returning a single mbuf to
112  *        UMA will typically be very fast vis a vis the cost of a memory
113  *        access.
114  */
115 
116 
117 /*
118  * File organization:
119  *  - private structures
120  *  - iflib private utility functions
121  *  - ifnet functions
122  *  - vlan registry and other exported functions
123  *  - iflib public core functions
124  *
125  *
126  */
127 static MALLOC_DEFINE(M_IFLIB, "iflib", "ifnet library");
128 
129 struct iflib_txq;
130 typedef struct iflib_txq *iflib_txq_t;
131 struct iflib_rxq;
132 typedef struct iflib_rxq *iflib_rxq_t;
133 struct iflib_fl;
134 typedef struct iflib_fl *iflib_fl_t;
135 
136 struct iflib_ctx;
137 
138 typedef struct iflib_filter_info {
139 	driver_filter_t *ifi_filter;
140 	void *ifi_filter_arg;
141 	struct grouptask *ifi_task;
142 	struct iflib_ctx *ifi_ctx;
143 } *iflib_filter_info_t;
144 
145 struct iflib_ctx {
146 	KOBJ_FIELDS;
147    /*
148    * Pointer to hardware driver's softc
149    */
150 	void *ifc_softc;
151 	device_t ifc_dev;
152 	if_t ifc_ifp;
153 
154 	cpuset_t ifc_cpus;
155 	if_shared_ctx_t ifc_sctx;
156 	struct if_softc_ctx ifc_softc_ctx;
157 
158 	struct mtx ifc_mtx;
159 
160 	uint16_t ifc_nhwtxqs;
161 	uint16_t ifc_nhwrxqs;
162 
163 	iflib_txq_t ifc_txqs;
164 	iflib_rxq_t ifc_rxqs;
165 	uint32_t ifc_if_flags;
166 	uint32_t ifc_flags;
167 	uint32_t ifc_max_fl_buf_size;
168 	int ifc_in_detach;
169 
170 	int ifc_link_state;
171 	int ifc_link_irq;
172 	int ifc_pause_frames;
173 	int ifc_watchdog_events;
174 	struct cdev *ifc_led_dev;
175 	struct resource *ifc_msix_mem;
176 
177 	struct if_irq ifc_legacy_irq;
178 	struct grouptask ifc_admin_task;
179 	struct grouptask ifc_vflr_task;
180 	struct iflib_filter_info ifc_filter_info;
181 	struct ifmedia	ifc_media;
182 
183 	struct sysctl_oid *ifc_sysctl_node;
184 	uint16_t ifc_sysctl_ntxqs;
185 	uint16_t ifc_sysctl_nrxqs;
186 	uint16_t ifc_sysctl_qs_eq_override;
187 
188 	uint16_t ifc_sysctl_ntxds[8];
189 	uint16_t ifc_sysctl_nrxds[8];
190 	struct if_txrx ifc_txrx;
191 #define isc_txd_encap  ifc_txrx.ift_txd_encap
192 #define isc_txd_flush  ifc_txrx.ift_txd_flush
193 #define isc_txd_credits_update  ifc_txrx.ift_txd_credits_update
194 #define isc_rxd_available ifc_txrx.ift_rxd_available
195 #define isc_rxd_pkt_get ifc_txrx.ift_rxd_pkt_get
196 #define isc_rxd_refill ifc_txrx.ift_rxd_refill
197 #define isc_rxd_flush ifc_txrx.ift_rxd_flush
198 #define isc_rxd_refill ifc_txrx.ift_rxd_refill
199 #define isc_rxd_refill ifc_txrx.ift_rxd_refill
200 #define isc_legacy_intr ifc_txrx.ift_legacy_intr
201 	eventhandler_tag ifc_vlan_attach_event;
202 	eventhandler_tag ifc_vlan_detach_event;
203 	uint8_t ifc_mac[ETHER_ADDR_LEN];
204 	char ifc_mtx_name[16];
205 };
206 
207 
208 void *
209 iflib_get_softc(if_ctx_t ctx)
210 {
211 
212 	return (ctx->ifc_softc);
213 }
214 
215 device_t
216 iflib_get_dev(if_ctx_t ctx)
217 {
218 
219 	return (ctx->ifc_dev);
220 }
221 
222 if_t
223 iflib_get_ifp(if_ctx_t ctx)
224 {
225 
226 	return (ctx->ifc_ifp);
227 }
228 
229 struct ifmedia *
230 iflib_get_media(if_ctx_t ctx)
231 {
232 
233 	return (&ctx->ifc_media);
234 }
235 
236 void
237 iflib_set_mac(if_ctx_t ctx, uint8_t mac[ETHER_ADDR_LEN])
238 {
239 
240 	bcopy(mac, ctx->ifc_mac, ETHER_ADDR_LEN);
241 }
242 
243 if_softc_ctx_t
244 iflib_get_softc_ctx(if_ctx_t ctx)
245 {
246 
247 	return (&ctx->ifc_softc_ctx);
248 }
249 
250 if_shared_ctx_t
251 iflib_get_sctx(if_ctx_t ctx)
252 {
253 
254 	return (ctx->ifc_sctx);
255 }
256 
257 #define CACHE_PTR_INCREMENT (CACHE_LINE_SIZE/sizeof(void*))
258 
259 #define LINK_ACTIVE(ctx) ((ctx)->ifc_link_state == LINK_STATE_UP)
260 #define CTX_IS_VF(ctx) ((ctx)->ifc_sctx->isc_flags & IFLIB_IS_VF)
261 
262 #define RX_SW_DESC_MAP_CREATED	(1 << 0)
263 #define TX_SW_DESC_MAP_CREATED	(1 << 1)
264 #define RX_SW_DESC_INUSE        (1 << 3)
265 #define TX_SW_DESC_MAPPED       (1 << 4)
266 
267 typedef struct iflib_sw_rx_desc_array {
268 	bus_dmamap_t	*ifsd_map;         /* bus_dma maps for packet */
269 	struct mbuf	**ifsd_m;           /* pkthdr mbufs */
270 	caddr_t		*ifsd_cl;          /* direct cluster pointer for rx */
271 	uint8_t		*ifsd_flags;
272 } iflib_rxsd_array_t;
273 
274 typedef struct iflib_sw_tx_desc_array {
275 	bus_dmamap_t    *ifsd_map;         /* bus_dma maps for packet */
276 	struct mbuf    **ifsd_m;           /* pkthdr mbufs */
277 	uint8_t		*ifsd_flags;
278 } iflib_txsd_array_t;
279 
280 
281 /* magic number that should be high enough for any hardware */
282 #define IFLIB_MAX_TX_SEGS		128
283 #define IFLIB_MAX_RX_SEGS		32
284 #define IFLIB_RX_COPY_THRESH		63
285 #define IFLIB_MAX_RX_REFRESH		32
286 #define IFLIB_QUEUE_IDLE		0
287 #define IFLIB_QUEUE_HUNG		1
288 #define IFLIB_QUEUE_WORKING		2
289 
290 /* this should really scale with ring size - 32 is a fairly arbitrary value for this */
291 #define TX_BATCH_SIZE			16
292 
293 #define IFLIB_RESTART_BUDGET		8
294 
295 #define	IFC_LEGACY		0x01
296 #define	IFC_QFLUSH		0x02
297 #define	IFC_MULTISEG		0x04
298 #define	IFC_DMAR		0x08
299 #define	IFC_SC_ALLOCATED	0x10
300 #define	IFC_INIT_DONE		0x20
301 
302 
303 #define CSUM_OFFLOAD		(CSUM_IP_TSO|CSUM_IP6_TSO|CSUM_IP| \
304 				 CSUM_IP_UDP|CSUM_IP_TCP|CSUM_IP_SCTP| \
305 				 CSUM_IP6_UDP|CSUM_IP6_TCP|CSUM_IP6_SCTP)
306 struct iflib_txq {
307 	uint16_t	ift_in_use;
308 	uint16_t	ift_cidx;
309 	uint16_t	ift_cidx_processed;
310 	uint16_t	ift_pidx;
311 	uint8_t		ift_gen;
312 	uint8_t		ift_db_pending;
313 	uint8_t		ift_db_pending_queued;
314 	uint8_t		ift_npending;
315 	uint8_t		ift_br_offset;
316 	/* implicit pad */
317 	uint64_t	ift_processed;
318 	uint64_t	ift_cleaned;
319 #if MEMORY_LOGGING
320 	uint64_t	ift_enqueued;
321 	uint64_t	ift_dequeued;
322 #endif
323 	uint64_t	ift_no_tx_dma_setup;
324 	uint64_t	ift_no_desc_avail;
325 	uint64_t	ift_mbuf_defrag_failed;
326 	uint64_t	ift_mbuf_defrag;
327 	uint64_t	ift_map_failed;
328 	uint64_t	ift_txd_encap_efbig;
329 	uint64_t	ift_pullups;
330 
331 	struct mtx	ift_mtx;
332 	struct mtx	ift_db_mtx;
333 
334 	/* constant values */
335 	if_ctx_t	ift_ctx;
336 	struct ifmp_ring        **ift_br;
337 	struct grouptask	ift_task;
338 	uint16_t	ift_size;
339 	uint16_t	ift_id;
340 	struct callout	ift_timer;
341 	struct callout	ift_db_check;
342 
343 	iflib_txsd_array_t	ift_sds;
344 	uint8_t			ift_nbr;
345 	uint8_t			ift_qstatus;
346 	uint8_t			ift_active;
347 	uint8_t			ift_closed;
348 	int			ift_watchdog_time;
349 	struct iflib_filter_info ift_filter_info;
350 	bus_dma_tag_t		ift_desc_tag;
351 	bus_dma_tag_t		ift_tso_desc_tag;
352 	iflib_dma_info_t	ift_ifdi;
353 #define MTX_NAME_LEN 16
354 	char                    ift_mtx_name[MTX_NAME_LEN];
355 	char                    ift_db_mtx_name[MTX_NAME_LEN];
356 	bus_dma_segment_t	ift_segs[IFLIB_MAX_TX_SEGS]  __aligned(CACHE_LINE_SIZE);
357 #ifdef IFLIB_DIAGNOSTICS
358 	uint64_t ift_cpu_exec_count[256];
359 #endif
360 } __aligned(CACHE_LINE_SIZE);
361 
362 struct iflib_fl {
363 	uint16_t	ifl_cidx;
364 	uint16_t	ifl_pidx;
365 	uint16_t	ifl_credits;
366 	uint8_t		ifl_gen;
367 #if MEMORY_LOGGING
368 	uint64_t	ifl_m_enqueued;
369 	uint64_t	ifl_m_dequeued;
370 	uint64_t	ifl_cl_enqueued;
371 	uint64_t	ifl_cl_dequeued;
372 #endif
373 	/* implicit pad */
374 
375 	/* constant */
376 	uint16_t	ifl_size;
377 	uint16_t	ifl_buf_size;
378 	uint16_t	ifl_cltype;
379 	uma_zone_t	ifl_zone;
380 	iflib_rxsd_array_t	ifl_sds;
381 	iflib_rxq_t	ifl_rxq;
382 	uint8_t		ifl_id;
383 	bus_dma_tag_t           ifl_desc_tag;
384 	iflib_dma_info_t	ifl_ifdi;
385 	uint64_t	ifl_bus_addrs[IFLIB_MAX_RX_REFRESH] __aligned(CACHE_LINE_SIZE);
386 	caddr_t		ifl_vm_addrs[IFLIB_MAX_RX_REFRESH];
387 }  __aligned(CACHE_LINE_SIZE);
388 
389 static inline int
390 get_inuse(int size, int cidx, int pidx, int gen)
391 {
392 	int used;
393 
394 	if (pidx > cidx)
395 		used = pidx - cidx;
396 	else if (pidx < cidx)
397 		used = size - cidx + pidx;
398 	else if (gen == 0 && pidx == cidx)
399 		used = 0;
400 	else if (gen == 1 && pidx == cidx)
401 		used = size;
402 	else
403 		panic("bad state");
404 
405 	return (used);
406 }
407 
408 #define TXQ_AVAIL(txq) (txq->ift_size - get_inuse(txq->ift_size, txq->ift_cidx, txq->ift_pidx, txq->ift_gen))
409 
410 #define IDXDIFF(head, tail, wrap) \
411 	((head) >= (tail) ? (head) - (tail) : (wrap) - (tail) + (head))
412 
413 struct iflib_rxq {
414 	/* If there is a separate completion queue -
415 	 * these are the cq cidx and pidx. Otherwise
416 	 * these are unused.
417 	 */
418 	uint16_t	ifr_size;
419 	uint16_t	ifr_cq_cidx;
420 	uint16_t	ifr_cq_pidx;
421 	uint8_t		ifr_cq_gen;
422 	uint8_t		ifr_fl_offset;
423 
424 	if_ctx_t	ifr_ctx;
425 	iflib_fl_t	ifr_fl;
426 	uint64_t	ifr_rx_irq;
427 	uint16_t	ifr_id;
428 	uint8_t		ifr_lro_enabled;
429 	uint8_t		ifr_nfl;
430 	struct lro_ctrl			ifr_lc;
431 	struct grouptask        ifr_task;
432 	struct iflib_filter_info ifr_filter_info;
433 	iflib_dma_info_t		ifr_ifdi;
434 	/* dynamically allocate if any drivers need a value substantially larger than this */
435 	struct if_rxd_frag	ifr_frags[IFLIB_MAX_RX_SEGS] __aligned(CACHE_LINE_SIZE);
436 #ifdef IFLIB_DIAGNOSTICS
437 	uint64_t ifr_cpu_exec_count[256];
438 #endif
439 }  __aligned(CACHE_LINE_SIZE);
440 
441 /*
442  * Only allow a single packet to take up most 1/nth of the tx ring
443  */
444 #define MAX_SINGLE_PACKET_FRACTION 12
445 #define IF_BAD_DMA (bus_addr_t)-1
446 
447 static int enable_msix = 1;
448 
449 #define CTX_ACTIVE(ctx) ((if_getdrvflags((ctx)->ifc_ifp) & IFF_DRV_RUNNING))
450 
451 #define CTX_LOCK_INIT(_sc, _name)  mtx_init(&(_sc)->ifc_mtx, _name, "iflib ctx lock", MTX_DEF)
452 
453 #define CTX_LOCK(ctx) mtx_lock(&(ctx)->ifc_mtx)
454 #define CTX_UNLOCK(ctx) mtx_unlock(&(ctx)->ifc_mtx)
455 #define CTX_LOCK_DESTROY(ctx) mtx_destroy(&(ctx)->ifc_mtx)
456 
457 
458 #define TXDB_LOCK_INIT(txq)  mtx_init(&(txq)->ift_db_mtx, (txq)->ift_db_mtx_name, NULL, MTX_DEF)
459 #define TXDB_TRYLOCK(txq) mtx_trylock(&(txq)->ift_db_mtx)
460 #define TXDB_LOCK(txq) mtx_lock(&(txq)->ift_db_mtx)
461 #define TXDB_UNLOCK(txq) mtx_unlock(&(txq)->ift_db_mtx)
462 #define TXDB_LOCK_DESTROY(txq) mtx_destroy(&(txq)->ift_db_mtx)
463 
464 #define CALLOUT_LOCK(txq)	mtx_lock(&txq->ift_mtx)
465 #define CALLOUT_UNLOCK(txq) 	mtx_unlock(&txq->ift_mtx)
466 
467 
468 /* Our boot-time initialization hook */
469 static int	iflib_module_event_handler(module_t, int, void *);
470 
471 static moduledata_t iflib_moduledata = {
472 	"iflib",
473 	iflib_module_event_handler,
474 	NULL
475 };
476 
477 DECLARE_MODULE(iflib, iflib_moduledata, SI_SUB_INIT_IF, SI_ORDER_ANY);
478 MODULE_VERSION(iflib, 1);
479 
480 MODULE_DEPEND(iflib, pci, 1, 1, 1);
481 MODULE_DEPEND(iflib, ether, 1, 1, 1);
482 
483 TASKQGROUP_DEFINE(if_io_tqg, mp_ncpus, 1);
484 TASKQGROUP_DEFINE(if_config_tqg, 1, 1);
485 
486 #ifndef IFLIB_DEBUG_COUNTERS
487 #ifdef INVARIANTS
488 #define IFLIB_DEBUG_COUNTERS 1
489 #else
490 #define IFLIB_DEBUG_COUNTERS 0
491 #endif /* !INVARIANTS */
492 #endif
493 
494 static SYSCTL_NODE(_net, OID_AUTO, iflib, CTLFLAG_RD, 0,
495                    "iflib driver parameters");
496 
497 /*
498  * XXX need to ensure that this can't accidentally cause the head to be moved backwards
499  */
500 static int iflib_min_tx_latency = 0;
501 
502 SYSCTL_INT(_net_iflib, OID_AUTO, min_tx_latency, CTLFLAG_RW,
503 		   &iflib_min_tx_latency, 0, "minimize transmit latency at the possible expense of throughput");
504 
505 
506 #if IFLIB_DEBUG_COUNTERS
507 
508 static int iflib_tx_seen;
509 static int iflib_tx_sent;
510 static int iflib_tx_encap;
511 static int iflib_rx_allocs;
512 static int iflib_fl_refills;
513 static int iflib_fl_refills_large;
514 static int iflib_tx_frees;
515 
516 SYSCTL_INT(_net_iflib, OID_AUTO, tx_seen, CTLFLAG_RD,
517 		   &iflib_tx_seen, 0, "# tx mbufs seen");
518 SYSCTL_INT(_net_iflib, OID_AUTO, tx_sent, CTLFLAG_RD,
519 		   &iflib_tx_sent, 0, "# tx mbufs sent");
520 SYSCTL_INT(_net_iflib, OID_AUTO, tx_encap, CTLFLAG_RD,
521 		   &iflib_tx_encap, 0, "# tx mbufs encapped");
522 SYSCTL_INT(_net_iflib, OID_AUTO, tx_frees, CTLFLAG_RD,
523 		   &iflib_tx_frees, 0, "# tx frees");
524 SYSCTL_INT(_net_iflib, OID_AUTO, rx_allocs, CTLFLAG_RD,
525 		   &iflib_rx_allocs, 0, "# rx allocations");
526 SYSCTL_INT(_net_iflib, OID_AUTO, fl_refills, CTLFLAG_RD,
527 		   &iflib_fl_refills, 0, "# refills");
528 SYSCTL_INT(_net_iflib, OID_AUTO, fl_refills_large, CTLFLAG_RD,
529 		   &iflib_fl_refills_large, 0, "# large refills");
530 
531 
532 static int iflib_txq_drain_flushing;
533 static int iflib_txq_drain_oactive;
534 static int iflib_txq_drain_notready;
535 static int iflib_txq_drain_encapfail;
536 
537 SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_flushing, CTLFLAG_RD,
538 		   &iflib_txq_drain_flushing, 0, "# drain flushes");
539 SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_oactive, CTLFLAG_RD,
540 		   &iflib_txq_drain_oactive, 0, "# drain oactives");
541 SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_notready, CTLFLAG_RD,
542 		   &iflib_txq_drain_notready, 0, "# drain notready");
543 SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_encapfail, CTLFLAG_RD,
544 		   &iflib_txq_drain_encapfail, 0, "# drain encap fails");
545 
546 
547 static int iflib_encap_load_mbuf_fail;
548 static int iflib_encap_txq_avail_fail;
549 static int iflib_encap_txd_encap_fail;
550 
551 SYSCTL_INT(_net_iflib, OID_AUTO, encap_load_mbuf_fail, CTLFLAG_RD,
552 		   &iflib_encap_load_mbuf_fail, 0, "# busdma load failures");
553 SYSCTL_INT(_net_iflib, OID_AUTO, encap_txq_avail_fail, CTLFLAG_RD,
554 		   &iflib_encap_txq_avail_fail, 0, "# txq avail failures");
555 SYSCTL_INT(_net_iflib, OID_AUTO, encap_txd_encap_fail, CTLFLAG_RD,
556 		   &iflib_encap_txd_encap_fail, 0, "# driver encap failures");
557 
558 static int iflib_task_fn_rxs;
559 static int iflib_rx_intr_enables;
560 static int iflib_fast_intrs;
561 static int iflib_intr_link;
562 static int iflib_intr_msix;
563 static int iflib_rx_unavail;
564 static int iflib_rx_ctx_inactive;
565 static int iflib_rx_zero_len;
566 static int iflib_rx_if_input;
567 static int iflib_rx_mbuf_null;
568 static int iflib_rxd_flush;
569 
570 static int iflib_verbose_debug;
571 
572 SYSCTL_INT(_net_iflib, OID_AUTO, intr_link, CTLFLAG_RD,
573 		   &iflib_intr_link, 0, "# intr link calls");
574 SYSCTL_INT(_net_iflib, OID_AUTO, intr_msix, CTLFLAG_RD,
575 		   &iflib_intr_msix, 0, "# intr msix calls");
576 SYSCTL_INT(_net_iflib, OID_AUTO, task_fn_rx, CTLFLAG_RD,
577 		   &iflib_task_fn_rxs, 0, "# task_fn_rx calls");
578 SYSCTL_INT(_net_iflib, OID_AUTO, rx_intr_enables, CTLFLAG_RD,
579 		   &iflib_rx_intr_enables, 0, "# rx intr enables");
580 SYSCTL_INT(_net_iflib, OID_AUTO, fast_intrs, CTLFLAG_RD,
581 		   &iflib_fast_intrs, 0, "# fast_intr calls");
582 SYSCTL_INT(_net_iflib, OID_AUTO, rx_unavail, CTLFLAG_RD,
583 		   &iflib_rx_unavail, 0, "# times rxeof called with no available data");
584 SYSCTL_INT(_net_iflib, OID_AUTO, rx_ctx_inactive, CTLFLAG_RD,
585 		   &iflib_rx_ctx_inactive, 0, "# times rxeof called with inactive context");
586 SYSCTL_INT(_net_iflib, OID_AUTO, rx_zero_len, CTLFLAG_RD,
587 		   &iflib_rx_zero_len, 0, "# times rxeof saw zero len mbuf");
588 SYSCTL_INT(_net_iflib, OID_AUTO, rx_if_input, CTLFLAG_RD,
589 		   &iflib_rx_if_input, 0, "# times rxeof called if_input");
590 SYSCTL_INT(_net_iflib, OID_AUTO, rx_mbuf_null, CTLFLAG_RD,
591 		   &iflib_rx_mbuf_null, 0, "# times rxeof got null mbuf");
592 SYSCTL_INT(_net_iflib, OID_AUTO, rxd_flush, CTLFLAG_RD,
593 	         &iflib_rxd_flush, 0, "# times rxd_flush called");
594 SYSCTL_INT(_net_iflib, OID_AUTO, verbose_debug, CTLFLAG_RW,
595 		   &iflib_verbose_debug, 0, "enable verbose debugging");
596 
597 #define DBG_COUNTER_INC(name) atomic_add_int(&(iflib_ ## name), 1)
598 static void
599 iflib_debug_reset(void)
600 {
601 	iflib_tx_seen = iflib_tx_sent = iflib_tx_encap = iflib_rx_allocs =
602 		iflib_fl_refills = iflib_fl_refills_large = iflib_tx_frees =
603 		iflib_txq_drain_flushing = iflib_txq_drain_oactive =
604 		iflib_txq_drain_notready = iflib_txq_drain_encapfail =
605 		iflib_encap_load_mbuf_fail = iflib_encap_txq_avail_fail =
606 		iflib_encap_txd_encap_fail = iflib_task_fn_rxs = iflib_rx_intr_enables =
607 		iflib_fast_intrs = iflib_intr_link = iflib_intr_msix = iflib_rx_unavail =
608 		iflib_rx_ctx_inactive = iflib_rx_zero_len = iflib_rx_if_input =
609 		iflib_rx_mbuf_null = iflib_rxd_flush = 0;
610 }
611 
612 #else
613 #define DBG_COUNTER_INC(name)
614 static void iflib_debug_reset(void) {}
615 #endif
616 
617 
618 
619 #define IFLIB_DEBUG 0
620 
621 static void iflib_tx_structures_free(if_ctx_t ctx);
622 static void iflib_rx_structures_free(if_ctx_t ctx);
623 static int iflib_queues_alloc(if_ctx_t ctx);
624 static int iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq);
625 static int iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, int cidx, int budget);
626 static int iflib_qset_structures_setup(if_ctx_t ctx);
627 static int iflib_msix_init(if_ctx_t ctx);
628 static int iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filterarg, int *rid, char *str);
629 static void iflib_txq_check_drain(iflib_txq_t txq, int budget);
630 static uint32_t iflib_txq_can_drain(struct ifmp_ring *);
631 static int iflib_register(if_ctx_t);
632 static void iflib_init_locked(if_ctx_t ctx);
633 static void iflib_add_device_sysctl_pre(if_ctx_t ctx);
634 static void iflib_add_device_sysctl_post(if_ctx_t ctx);
635 static void iflib_ifmp_purge(iflib_txq_t txq);
636 static void _iflib_pre_assert(if_softc_ctx_t scctx);
637 
638 #ifdef DEV_NETMAP
639 #include <sys/selinfo.h>
640 #include <net/netmap.h>
641 #include <dev/netmap/netmap_kern.h>
642 
643 MODULE_DEPEND(iflib, netmap, 1, 1, 1);
644 
645 /*
646  * device-specific sysctl variables:
647  *
648  * iflib_crcstrip: 0: keep CRC in rx frames (default), 1: strip it.
649  *	During regular operations the CRC is stripped, but on some
650  *	hardware reception of frames not multiple of 64 is slower,
651  *	so using crcstrip=0 helps in benchmarks.
652  *
653  * iflib_rx_miss, iflib_rx_miss_bufs:
654  *	count packets that might be missed due to lost interrupts.
655  */
656 SYSCTL_DECL(_dev_netmap);
657 /*
658  * The xl driver by default strips CRCs and we do not override it.
659  */
660 
661 int iflib_crcstrip = 1;
662 SYSCTL_INT(_dev_netmap, OID_AUTO, iflib_crcstrip,
663     CTLFLAG_RW, &iflib_crcstrip, 1, "strip CRC on rx frames");
664 
665 int iflib_rx_miss, iflib_rx_miss_bufs;
666 SYSCTL_INT(_dev_netmap, OID_AUTO, iflib_rx_miss,
667     CTLFLAG_RW, &iflib_rx_miss, 0, "potentially missed rx intr");
668 SYSCTL_INT(_dev_netmap, OID_AUTO, iflib_rx_miss_bufs,
669     CTLFLAG_RW, &iflib_rx_miss_bufs, 0, "potentially missed rx intr bufs");
670 
671 /*
672  * Register/unregister. We are already under netmap lock.
673  * Only called on the first register or the last unregister.
674  */
675 static int
676 iflib_netmap_register(struct netmap_adapter *na, int onoff)
677 {
678 	struct ifnet *ifp = na->ifp;
679 	if_ctx_t ctx = ifp->if_softc;
680 
681 	CTX_LOCK(ctx);
682 	IFDI_INTR_DISABLE(ctx);
683 
684 	/* Tell the stack that the interface is no longer active */
685 	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
686 
687 	if (!CTX_IS_VF(ctx))
688 		IFDI_CRCSTRIP_SET(ctx, onoff, iflib_crcstrip);
689 
690 	/* enable or disable flags and callbacks in na and ifp */
691 	if (onoff) {
692 		nm_set_native_flags(na);
693 	} else {
694 		nm_clear_native_flags(na);
695 	}
696 	IFDI_INIT(ctx);
697 	IFDI_CRCSTRIP_SET(ctx, onoff, iflib_crcstrip); // XXX why twice ?
698 	CTX_UNLOCK(ctx);
699 	return (ifp->if_drv_flags & IFF_DRV_RUNNING ? 0 : 1);
700 }
701 
702 /*
703  * Reconcile kernel and user view of the transmit ring.
704  *
705  * All information is in the kring.
706  * Userspace wants to send packets up to the one before kring->rhead,
707  * kernel knows kring->nr_hwcur is the first unsent packet.
708  *
709  * Here we push packets out (as many as possible), and possibly
710  * reclaim buffers from previously completed transmission.
711  *
712  * The caller (netmap) guarantees that there is only one instance
713  * running at any time. Any interference with other driver
714  * methods should be handled by the individual drivers.
715  */
716 static int
717 iflib_netmap_txsync(struct netmap_kring *kring, int flags)
718 {
719 	struct netmap_adapter *na = kring->na;
720 	struct ifnet *ifp = na->ifp;
721 	struct netmap_ring *ring = kring->ring;
722 	u_int nm_i;	/* index into the netmap ring */
723 	u_int nic_i;	/* index into the NIC ring */
724 	u_int n;
725 	u_int const lim = kring->nkr_num_slots - 1;
726 	u_int const head = kring->rhead;
727 	struct if_pkt_info pi;
728 
729 	/*
730 	 * interrupts on every tx packet are expensive so request
731 	 * them every half ring, or where NS_REPORT is set
732 	 */
733 	u_int report_frequency = kring->nkr_num_slots >> 1;
734 	/* device-specific */
735 	if_ctx_t ctx = ifp->if_softc;
736 	iflib_txq_t txq = &ctx->ifc_txqs[kring->ring_id];
737 
738 	pi.ipi_segs = txq->ift_segs;
739 	pi.ipi_qsidx = kring->ring_id;
740 	pi.ipi_ndescs = 0;
741 
742 	bus_dmamap_sync(txq->ift_desc_tag, txq->ift_ifdi->idi_map,
743 					BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
744 
745 
746 	/*
747 	 * First part: process new packets to send.
748 	 * nm_i is the current index in the netmap ring,
749 	 * nic_i is the corresponding index in the NIC ring.
750 	 *
751 	 * If we have packets to send (nm_i != head)
752 	 * iterate over the netmap ring, fetch length and update
753 	 * the corresponding slot in the NIC ring. Some drivers also
754 	 * need to update the buffer's physical address in the NIC slot
755 	 * even NS_BUF_CHANGED is not set (PNMB computes the addresses).
756 	 *
757 	 * The netmap_reload_map() calls is especially expensive,
758 	 * even when (as in this case) the tag is 0, so do only
759 	 * when the buffer has actually changed.
760 	 *
761 	 * If possible do not set the report/intr bit on all slots,
762 	 * but only a few times per ring or when NS_REPORT is set.
763 	 *
764 	 * Finally, on 10G and faster drivers, it might be useful
765 	 * to prefetch the next slot and txr entry.
766 	 */
767 
768 	nm_i = kring->nr_hwcur;
769 	if (nm_i != head) {	/* we have new packets to send */
770 		nic_i = netmap_idx_k2n(kring, nm_i);
771 
772 		__builtin_prefetch(&ring->slot[nm_i]);
773 		__builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i]);
774 		__builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i]);
775 
776 		for (n = 0; nm_i != head; n++) {
777 			struct netmap_slot *slot = &ring->slot[nm_i];
778 			u_int len = slot->len;
779 			uint64_t paddr;
780 			void *addr = PNMB(na, slot, &paddr);
781 			int flags = (slot->flags & NS_REPORT ||
782 				nic_i == 0 || nic_i == report_frequency) ?
783 				IPI_TX_INTR : 0;
784 
785 			/* device-specific */
786 			pi.ipi_pidx = nic_i;
787 			pi.ipi_flags = flags;
788 
789 			/* Fill the slot in the NIC ring. */
790 			ctx->isc_txd_encap(ctx->ifc_softc, &pi);
791 
792 			/* prefetch for next round */
793 			__builtin_prefetch(&ring->slot[nm_i + 1]);
794 			__builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i + 1]);
795 			__builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i + 1]);
796 
797 			NM_CHECK_ADDR_LEN(na, addr, len);
798 
799 			if (slot->flags & NS_BUF_CHANGED) {
800 				/* buffer has changed, reload map */
801 				netmap_reload_map(na, txq->ift_desc_tag, txq->ift_sds.ifsd_map[nic_i], addr);
802 			}
803 			slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED);
804 
805 			/* make sure changes to the buffer are synced */
806 			bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_sds.ifsd_map[nic_i],
807 							BUS_DMASYNC_PREWRITE);
808 
809 			nm_i = nm_next(nm_i, lim);
810 			nic_i = nm_next(nic_i, lim);
811 		}
812 		kring->nr_hwcur = head;
813 
814 		/* synchronize the NIC ring */
815 		bus_dmamap_sync(txq->ift_desc_tag, txq->ift_ifdi->idi_map,
816 						BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
817 
818 		/* (re)start the tx unit up to slot nic_i (excluded) */
819 		ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, nic_i);
820 	}
821 
822 	/*
823 	 * Second part: reclaim buffers for completed transmissions.
824 	 */
825 	if (iflib_tx_credits_update(ctx, txq)) {
826 		/* some tx completed, increment avail */
827 		nic_i = txq->ift_cidx_processed;
828 		kring->nr_hwtail = nm_prev(netmap_idx_n2k(kring, nic_i), lim);
829 	}
830 	return (0);
831 }
832 
833 /*
834  * Reconcile kernel and user view of the receive ring.
835  * Same as for the txsync, this routine must be efficient.
836  * The caller guarantees a single invocations, but races against
837  * the rest of the driver should be handled here.
838  *
839  * On call, kring->rhead is the first packet that userspace wants
840  * to keep, and kring->rcur is the wakeup point.
841  * The kernel has previously reported packets up to kring->rtail.
842  *
843  * If (flags & NAF_FORCE_READ) also check for incoming packets irrespective
844  * of whether or not we received an interrupt.
845  */
846 static int
847 iflib_netmap_rxsync(struct netmap_kring *kring, int flags)
848 {
849 	struct netmap_adapter *na = kring->na;
850 	struct ifnet *ifp = na->ifp;
851 	struct netmap_ring *ring = kring->ring;
852 	u_int nm_i;	/* index into the netmap ring */
853 	u_int nic_i;	/* index into the NIC ring */
854 	u_int i, n;
855 	u_int const lim = kring->nkr_num_slots - 1;
856 	u_int const head = kring->rhead;
857 	int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;
858 	struct if_rxd_info ri;
859 	/* device-specific */
860 	if_ctx_t ctx = ifp->if_softc;
861 	iflib_rxq_t rxq = &ctx->ifc_rxqs[kring->ring_id];
862 	iflib_fl_t fl = rxq->ifr_fl;
863 	if (head > lim)
864 		return netmap_ring_reinit(kring);
865 
866 	bzero(&ri, sizeof(ri));
867 	ri.iri_qsidx = kring->ring_id;
868 	ri.iri_ifp = ctx->ifc_ifp;
869 	/* XXX check sync modes */
870 	for (i = 0, fl = rxq->ifr_fl; i < rxq->ifr_nfl; i++, fl++)
871 		bus_dmamap_sync(rxq->ifr_fl[i].ifl_desc_tag, fl->ifl_ifdi->idi_map,
872 				BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
873 
874 	/*
875 	 * First part: import newly received packets.
876 	 *
877 	 * nm_i is the index of the next free slot in the netmap ring,
878 	 * nic_i is the index of the next received packet in the NIC ring,
879 	 * and they may differ in case if_init() has been called while
880 	 * in netmap mode. For the receive ring we have
881 	 *
882 	 *	nic_i = rxr->next_check;
883 	 *	nm_i = kring->nr_hwtail (previous)
884 	 * and
885 	 *	nm_i == (nic_i + kring->nkr_hwofs) % ring_size
886 	 *
887 	 * rxr->next_check is set to 0 on a ring reinit
888 	 */
889 	if (netmap_no_pendintr || force_update) {
890 		int crclen = iflib_crcstrip ? 0 : 4;
891 		int error, avail;
892 		uint16_t slot_flags = kring->nkr_slot_flags;
893 
894 		for (fl = rxq->ifr_fl, i = 0; i < rxq->ifr_nfl; i++, fl++) {
895 			nic_i = fl->ifl_cidx;
896 			nm_i = netmap_idx_n2k(kring, nic_i);
897 			avail = ctx->isc_rxd_available(ctx->ifc_softc, kring->ring_id, nic_i, INT_MAX);
898 			for (n = 0; avail > 0; n++, avail--) {
899 				error = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri);
900 				if (error)
901 					ring->slot[nm_i].len = 0;
902 				else
903 					ring->slot[nm_i].len = ri.iri_len - crclen;
904 				ring->slot[nm_i].flags = slot_flags;
905 				bus_dmamap_sync(fl->ifl_ifdi->idi_tag,
906 								fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD);
907 				nm_i = nm_next(nm_i, lim);
908 				nic_i = nm_next(nic_i, lim);
909 			}
910 			if (n) { /* update the state variables */
911 				if (netmap_no_pendintr && !force_update) {
912 					/* diagnostics */
913 					iflib_rx_miss ++;
914 					iflib_rx_miss_bufs += n;
915 				}
916 				fl->ifl_cidx = nic_i;
917 				kring->nr_hwtail = nm_i;
918 			}
919 			kring->nr_kflags &= ~NKR_PENDINTR;
920 		}
921 	}
922 	/*
923 	 * Second part: skip past packets that userspace has released.
924 	 * (kring->nr_hwcur to head excluded),
925 	 * and make the buffers available for reception.
926 	 * As usual nm_i is the index in the netmap ring,
927 	 * nic_i is the index in the NIC ring, and
928 	 * nm_i == (nic_i + kring->nkr_hwofs) % ring_size
929 	 */
930 	/* XXX not sure how this will work with multiple free lists */
931 	nm_i = kring->nr_hwcur;
932 	if (nm_i != head) {
933 		nic_i = netmap_idx_k2n(kring, nm_i);
934 		for (n = 0; nm_i != head; n++) {
935 			struct netmap_slot *slot = &ring->slot[nm_i];
936 			uint64_t paddr;
937 			caddr_t vaddr;
938 			void *addr = PNMB(na, slot, &paddr);
939 
940 			if (addr == NETMAP_BUF_BASE(na)) /* bad buf */
941 				goto ring_reset;
942 
943 			vaddr = addr;
944 			if (slot->flags & NS_BUF_CHANGED) {
945 				/* buffer has changed, reload map */
946 				netmap_reload_map(na, fl->ifl_ifdi->idi_tag, fl->ifl_sds.ifsd_map[nic_i], addr);
947 				slot->flags &= ~NS_BUF_CHANGED;
948 			}
949 			/*
950 			 * XXX we should be batching this operation - TODO
951 			 */
952 			ctx->isc_rxd_refill(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, nic_i, &paddr, &vaddr, 1, fl->ifl_buf_size);
953 			bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_sds.ifsd_map[nic_i],
954 			    BUS_DMASYNC_PREREAD);
955 			nm_i = nm_next(nm_i, lim);
956 			nic_i = nm_next(nic_i, lim);
957 		}
958 		kring->nr_hwcur = head;
959 
960 		bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
961 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
962 		/*
963 		 * IMPORTANT: we must leave one free slot in the ring,
964 		 * so move nic_i back by one unit
965 		 */
966 		nic_i = nm_prev(nic_i, lim);
967 		ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, nic_i);
968 	}
969 
970 	return 0;
971 
972 ring_reset:
973 	return netmap_ring_reinit(kring);
974 }
975 
976 static int
977 iflib_netmap_attach(if_ctx_t ctx)
978 {
979 	struct netmap_adapter na;
980 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
981 
982 	bzero(&na, sizeof(na));
983 
984 	na.ifp = ctx->ifc_ifp;
985 	na.na_flags = NAF_BDG_MAYSLEEP;
986 	MPASS(ctx->ifc_softc_ctx.isc_ntxqsets);
987 	MPASS(ctx->ifc_softc_ctx.isc_nrxqsets);
988 
989 	na.num_tx_desc = scctx->isc_ntxd[0];
990 	na.num_rx_desc = scctx->isc_nrxd[0];
991 	na.nm_txsync = iflib_netmap_txsync;
992 	na.nm_rxsync = iflib_netmap_rxsync;
993 	na.nm_register = iflib_netmap_register;
994 	na.num_tx_rings = ctx->ifc_softc_ctx.isc_ntxqsets;
995 	na.num_rx_rings = ctx->ifc_softc_ctx.isc_nrxqsets;
996 	return (netmap_attach(&na));
997 }
998 
999 static void
1000 iflib_netmap_txq_init(if_ctx_t ctx, iflib_txq_t txq)
1001 {
1002 	struct netmap_adapter *na = NA(ctx->ifc_ifp);
1003 	struct netmap_slot *slot;
1004 
1005 	slot = netmap_reset(na, NR_TX, txq->ift_id, 0);
1006 	if (slot == 0)
1007 		return;
1008 
1009 	for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxd[0]; i++) {
1010 
1011 		/*
1012 		 * In netmap mode, set the map for the packet buffer.
1013 		 * NOTE: Some drivers (not this one) also need to set
1014 		 * the physical buffer address in the NIC ring.
1015 		 * netmap_idx_n2k() maps a nic index, i, into the corresponding
1016 		 * netmap slot index, si
1017 		 */
1018 		int si = netmap_idx_n2k(&na->tx_rings[txq->ift_id], i);
1019 		netmap_load_map(na, txq->ift_desc_tag, txq->ift_sds.ifsd_map[i], NMB(na, slot + si));
1020 	}
1021 }
1022 static void
1023 iflib_netmap_rxq_init(if_ctx_t ctx, iflib_rxq_t rxq)
1024 {
1025 	struct netmap_adapter *na = NA(ctx->ifc_ifp);
1026 	struct netmap_slot *slot;
1027 	bus_dmamap_t *map;
1028 	int nrxd;
1029 
1030 	slot = netmap_reset(na, NR_RX, rxq->ifr_id, 0);
1031 	if (slot == 0)
1032 		return;
1033 	map = rxq->ifr_fl[0].ifl_sds.ifsd_map;
1034 	nrxd = ctx->ifc_softc_ctx.isc_nrxd[0];
1035 	for (int i = 0; i < nrxd; i++, map++) {
1036 			int sj = netmap_idx_n2k(&na->rx_rings[rxq->ifr_id], i);
1037 			uint64_t paddr;
1038 			void *addr;
1039 			caddr_t vaddr;
1040 
1041 			vaddr = addr = PNMB(na, slot + sj, &paddr);
1042 			netmap_load_map(na, rxq->ifr_fl[0].ifl_ifdi->idi_tag, *map, addr);
1043 			/* Update descriptor and the cached value */
1044 			ctx->isc_rxd_refill(ctx->ifc_softc, rxq->ifr_id, 0 /* fl_id */, i, &paddr, &vaddr, 1, rxq->ifr_fl[0].ifl_buf_size);
1045 	}
1046 	/* preserve queue */
1047 	if (ctx->ifc_ifp->if_capenable & IFCAP_NETMAP) {
1048 		struct netmap_kring *kring = &na->rx_rings[rxq->ifr_id];
1049 		int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
1050 		ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, 0 /* fl_id */, t);
1051 	} else
1052 		ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, 0 /* fl_id */, nrxd-1);
1053 }
1054 
1055 #define iflib_netmap_detach(ifp) netmap_detach(ifp)
1056 
1057 #else
1058 #define iflib_netmap_txq_init(ctx, txq)
1059 #define iflib_netmap_rxq_init(ctx, rxq)
1060 #define iflib_netmap_detach(ifp)
1061 
1062 #define iflib_netmap_attach(ctx) (0)
1063 #define netmap_rx_irq(ifp, qid, budget) (0)
1064 
1065 #endif
1066 
1067 #if defined(__i386__) || defined(__amd64__)
1068 static __inline void
1069 prefetch(void *x)
1070 {
1071 	__asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x));
1072 }
1073 #else
1074 #define prefetch(x)
1075 #endif
1076 
1077 static void
1078 _iflib_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err)
1079 {
1080 	if (err)
1081 		return;
1082 	*(bus_addr_t *) arg = segs[0].ds_addr;
1083 }
1084 
1085 int
1086 iflib_dma_alloc(if_ctx_t ctx, int size, iflib_dma_info_t dma, int mapflags)
1087 {
1088 	int err;
1089 	if_shared_ctx_t sctx = ctx->ifc_sctx;
1090 	device_t dev = ctx->ifc_dev;
1091 
1092 	KASSERT(sctx->isc_q_align != 0, ("alignment value not initialized"));
1093 
1094 	err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
1095 				sctx->isc_q_align, 0,	/* alignment, bounds */
1096 				BUS_SPACE_MAXADDR,	/* lowaddr */
1097 				BUS_SPACE_MAXADDR,	/* highaddr */
1098 				NULL, NULL,		/* filter, filterarg */
1099 				size,			/* maxsize */
1100 				1,			/* nsegments */
1101 				size,			/* maxsegsize */
1102 				BUS_DMA_ALLOCNOW,	/* flags */
1103 				NULL,			/* lockfunc */
1104 				NULL,			/* lockarg */
1105 				&dma->idi_tag);
1106 	if (err) {
1107 		device_printf(dev,
1108 		    "%s: bus_dma_tag_create failed: %d\n",
1109 		    __func__, err);
1110 		goto fail_0;
1111 	}
1112 
1113 	err = bus_dmamem_alloc(dma->idi_tag, (void**) &dma->idi_vaddr,
1114 	    BUS_DMA_NOWAIT | BUS_DMA_COHERENT | BUS_DMA_ZERO, &dma->idi_map);
1115 	if (err) {
1116 		device_printf(dev,
1117 		    "%s: bus_dmamem_alloc(%ju) failed: %d\n",
1118 		    __func__, (uintmax_t)size, err);
1119 		goto fail_1;
1120 	}
1121 
1122 	dma->idi_paddr = IF_BAD_DMA;
1123 	err = bus_dmamap_load(dma->idi_tag, dma->idi_map, dma->idi_vaddr,
1124 	    size, _iflib_dmamap_cb, &dma->idi_paddr, mapflags | BUS_DMA_NOWAIT);
1125 	if (err || dma->idi_paddr == IF_BAD_DMA) {
1126 		device_printf(dev,
1127 		    "%s: bus_dmamap_load failed: %d\n",
1128 		    __func__, err);
1129 		goto fail_2;
1130 	}
1131 
1132 	dma->idi_size = size;
1133 	return (0);
1134 
1135 fail_2:
1136 	bus_dmamem_free(dma->idi_tag, dma->idi_vaddr, dma->idi_map);
1137 fail_1:
1138 	bus_dma_tag_destroy(dma->idi_tag);
1139 fail_0:
1140 	dma->idi_tag = NULL;
1141 
1142 	return (err);
1143 }
1144 
1145 int
1146 iflib_dma_alloc_multi(if_ctx_t ctx, int *sizes, iflib_dma_info_t *dmalist, int mapflags, int count)
1147 {
1148 	int i, err;
1149 	iflib_dma_info_t *dmaiter;
1150 
1151 	dmaiter = dmalist;
1152 	for (i = 0; i < count; i++, dmaiter++) {
1153 		if ((err = iflib_dma_alloc(ctx, sizes[i], *dmaiter, mapflags)) != 0)
1154 			break;
1155 	}
1156 	if (err)
1157 		iflib_dma_free_multi(dmalist, i);
1158 	return (err);
1159 }
1160 
1161 void
1162 iflib_dma_free(iflib_dma_info_t dma)
1163 {
1164 	if (dma->idi_tag == NULL)
1165 		return;
1166 	if (dma->idi_paddr != IF_BAD_DMA) {
1167 		bus_dmamap_sync(dma->idi_tag, dma->idi_map,
1168 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1169 		bus_dmamap_unload(dma->idi_tag, dma->idi_map);
1170 		dma->idi_paddr = IF_BAD_DMA;
1171 	}
1172 	if (dma->idi_vaddr != NULL) {
1173 		bus_dmamem_free(dma->idi_tag, dma->idi_vaddr, dma->idi_map);
1174 		dma->idi_vaddr = NULL;
1175 	}
1176 	bus_dma_tag_destroy(dma->idi_tag);
1177 	dma->idi_tag = NULL;
1178 }
1179 
1180 void
1181 iflib_dma_free_multi(iflib_dma_info_t *dmalist, int count)
1182 {
1183 	int i;
1184 	iflib_dma_info_t *dmaiter = dmalist;
1185 
1186 	for (i = 0; i < count; i++, dmaiter++)
1187 		iflib_dma_free(*dmaiter);
1188 }
1189 
1190 #ifdef EARLY_AP_STARTUP
1191 static const int iflib_started = 1;
1192 #else
1193 /*
1194  * We used to abuse the smp_started flag to decide if the queues have been
1195  * fully initialized (by late taskqgroup_adjust() calls in a SYSINIT()).
1196  * That gave bad races, since the SYSINIT() runs strictly after smp_started
1197  * is set.  Run a SYSINIT() strictly after that to just set a usable
1198  * completion flag.
1199  */
1200 
1201 static int iflib_started;
1202 
1203 static void
1204 iflib_record_started(void *arg)
1205 {
1206 	iflib_started = 1;
1207 }
1208 
1209 SYSINIT(iflib_record_started, SI_SUB_SMP + 1, SI_ORDER_FIRST,
1210 	iflib_record_started, NULL);
1211 #endif
1212 
1213 static int
1214 iflib_fast_intr(void *arg)
1215 {
1216 	iflib_filter_info_t info = arg;
1217 	struct grouptask *gtask = info->ifi_task;
1218 
1219 	if (!iflib_started)
1220 		return (FILTER_HANDLED);
1221 
1222 	DBG_COUNTER_INC(fast_intrs);
1223 	if (info->ifi_filter != NULL && info->ifi_filter(info->ifi_filter_arg) == FILTER_HANDLED)
1224 		return (FILTER_HANDLED);
1225 
1226 	GROUPTASK_ENQUEUE(gtask);
1227 	return (FILTER_HANDLED);
1228 }
1229 
1230 static int
1231 _iflib_irq_alloc(if_ctx_t ctx, if_irq_t irq, int rid,
1232 	driver_filter_t filter, driver_intr_t handler, void *arg,
1233 				 char *name)
1234 {
1235 	int rc;
1236 	struct resource *res;
1237 	void *tag;
1238 	device_t dev = ctx->ifc_dev;
1239 
1240 	MPASS(rid < 512);
1241 	irq->ii_rid = rid;
1242 	res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &irq->ii_rid,
1243 				     RF_SHAREABLE | RF_ACTIVE);
1244 	if (res == NULL) {
1245 		device_printf(dev,
1246 		    "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
1247 		return (ENOMEM);
1248 	}
1249 	irq->ii_res = res;
1250 	KASSERT(filter == NULL || handler == NULL, ("filter and handler can't both be non-NULL"));
1251 	rc = bus_setup_intr(dev, res, INTR_MPSAFE | INTR_TYPE_NET,
1252 						filter, handler, arg, &tag);
1253 	if (rc != 0) {
1254 		device_printf(dev,
1255 		    "failed to setup interrupt for rid %d, name %s: %d\n",
1256 					  rid, name ? name : "unknown", rc);
1257 		return (rc);
1258 	} else if (name)
1259 		bus_describe_intr(dev, res, tag, "%s", name);
1260 
1261 	irq->ii_tag = tag;
1262 	return (0);
1263 }
1264 
1265 
1266 /*********************************************************************
1267  *
1268  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
1269  *  the information needed to transmit a packet on the wire. This is
1270  *  called only once at attach, setup is done every reset.
1271  *
1272  **********************************************************************/
1273 
1274 static int
1275 iflib_txsd_alloc(iflib_txq_t txq)
1276 {
1277 	if_ctx_t ctx = txq->ift_ctx;
1278 	if_shared_ctx_t sctx = ctx->ifc_sctx;
1279 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
1280 	device_t dev = ctx->ifc_dev;
1281 	int err, nsegments, ntsosegments;
1282 
1283 	nsegments = scctx->isc_tx_nsegments;
1284 	ntsosegments = scctx->isc_tx_tso_segments_max;
1285 	MPASS(scctx->isc_ntxd[0] > 0);
1286 	MPASS(scctx->isc_ntxd[txq->ift_br_offset] > 0);
1287 	MPASS(nsegments > 0);
1288 	MPASS(ntsosegments > 0);
1289 	/*
1290 	 * Setup DMA descriptor areas.
1291 	 */
1292 	if ((err = bus_dma_tag_create(bus_get_dma_tag(dev),
1293 			       1, 0,			/* alignment, bounds */
1294 			       BUS_SPACE_MAXADDR,	/* lowaddr */
1295 			       BUS_SPACE_MAXADDR,	/* highaddr */
1296 			       NULL, NULL,		/* filter, filterarg */
1297 			       sctx->isc_tx_maxsize,		/* maxsize */
1298 			       nsegments,	/* nsegments */
1299 			       sctx->isc_tx_maxsegsize,	/* maxsegsize */
1300 			       0,			/* flags */
1301 			       NULL,			/* lockfunc */
1302 			       NULL,			/* lockfuncarg */
1303 			       &txq->ift_desc_tag))) {
1304 		device_printf(dev,"Unable to allocate TX DMA tag: %d\n", err);
1305 		device_printf(dev,"maxsize: %zd nsegments: %d maxsegsize: %zd\n",
1306 					  sctx->isc_tx_maxsize, nsegments, sctx->isc_tx_maxsegsize);
1307 		goto fail;
1308 	}
1309 	if ((err = bus_dma_tag_create(bus_get_dma_tag(dev),
1310 			       1, 0,			/* alignment, bounds */
1311 			       BUS_SPACE_MAXADDR,	/* lowaddr */
1312 			       BUS_SPACE_MAXADDR,	/* highaddr */
1313 			       NULL, NULL,		/* filter, filterarg */
1314 			       scctx->isc_tx_tso_size_max,		/* maxsize */
1315 			       ntsosegments,	/* nsegments */
1316 			       scctx->isc_tx_tso_segsize_max,	/* maxsegsize */
1317 			       0,			/* flags */
1318 			       NULL,			/* lockfunc */
1319 			       NULL,			/* lockfuncarg */
1320 			       &txq->ift_tso_desc_tag))) {
1321 		device_printf(dev,"Unable to allocate TX TSO DMA tag: %d\n", err);
1322 
1323 		goto fail;
1324 	}
1325 	if (!(txq->ift_sds.ifsd_flags =
1326 	    (uint8_t *) malloc(sizeof(uint8_t) *
1327 	    scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
1328 		device_printf(dev, "Unable to allocate tx_buffer memory\n");
1329 		err = ENOMEM;
1330 		goto fail;
1331 	}
1332 	if (!(txq->ift_sds.ifsd_m =
1333 	    (struct mbuf **) malloc(sizeof(struct mbuf *) *
1334 	    scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
1335 		device_printf(dev, "Unable to allocate tx_buffer memory\n");
1336 		err = ENOMEM;
1337 		goto fail;
1338 	}
1339 
1340         /* Create the descriptor buffer dma maps */
1341 #if defined(ACPI_DMAR) || (!(defined(__i386__) && !defined(__amd64__)))
1342 	if ((ctx->ifc_flags & IFC_DMAR) == 0)
1343 		return (0);
1344 
1345 	if (!(txq->ift_sds.ifsd_map =
1346 	    (bus_dmamap_t *) malloc(sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
1347 		device_printf(dev, "Unable to allocate tx_buffer map memory\n");
1348 		err = ENOMEM;
1349 		goto fail;
1350 	}
1351 
1352 	for (int i = 0; i < scctx->isc_ntxd[txq->ift_br_offset]; i++) {
1353 		err = bus_dmamap_create(txq->ift_desc_tag, 0, &txq->ift_sds.ifsd_map[i]);
1354 		if (err != 0) {
1355 			device_printf(dev, "Unable to create TX DMA map\n");
1356 			goto fail;
1357 		}
1358 	}
1359 #endif
1360 	return (0);
1361 fail:
1362 	/* We free all, it handles case where we are in the middle */
1363 	iflib_tx_structures_free(ctx);
1364 	return (err);
1365 }
1366 
1367 static void
1368 iflib_txsd_destroy(if_ctx_t ctx, iflib_txq_t txq, int i)
1369 {
1370 	bus_dmamap_t map;
1371 
1372 	map = NULL;
1373 	if (txq->ift_sds.ifsd_map != NULL)
1374 		map = txq->ift_sds.ifsd_map[i];
1375 	if (map != NULL) {
1376 		bus_dmamap_unload(txq->ift_desc_tag, map);
1377 		bus_dmamap_destroy(txq->ift_desc_tag, map);
1378 		txq->ift_sds.ifsd_map[i] = NULL;
1379 	}
1380 }
1381 
1382 static void
1383 iflib_txq_destroy(iflib_txq_t txq)
1384 {
1385 	if_ctx_t ctx = txq->ift_ctx;
1386 
1387 	for (int i = 0; i < txq->ift_size; i++)
1388 		iflib_txsd_destroy(ctx, txq, i);
1389 	if (txq->ift_sds.ifsd_map != NULL) {
1390 		free(txq->ift_sds.ifsd_map, M_IFLIB);
1391 		txq->ift_sds.ifsd_map = NULL;
1392 	}
1393 	if (txq->ift_sds.ifsd_m != NULL) {
1394 		free(txq->ift_sds.ifsd_m, M_IFLIB);
1395 		txq->ift_sds.ifsd_m = NULL;
1396 	}
1397 	if (txq->ift_sds.ifsd_flags != NULL) {
1398 		free(txq->ift_sds.ifsd_flags, M_IFLIB);
1399 		txq->ift_sds.ifsd_flags = NULL;
1400 	}
1401 	if (txq->ift_desc_tag != NULL) {
1402 		bus_dma_tag_destroy(txq->ift_desc_tag);
1403 		txq->ift_desc_tag = NULL;
1404 	}
1405 	if (txq->ift_tso_desc_tag != NULL) {
1406 		bus_dma_tag_destroy(txq->ift_tso_desc_tag);
1407 		txq->ift_tso_desc_tag = NULL;
1408 	}
1409 }
1410 
1411 static void
1412 iflib_txsd_free(if_ctx_t ctx, iflib_txq_t txq, int i)
1413 {
1414 	struct mbuf **mp;
1415 
1416 	mp = &txq->ift_sds.ifsd_m[i];
1417 	if (*mp == NULL)
1418 		return;
1419 
1420 	if (txq->ift_sds.ifsd_map != NULL) {
1421 		bus_dmamap_sync(txq->ift_desc_tag,
1422 				txq->ift_sds.ifsd_map[i],
1423 				BUS_DMASYNC_POSTWRITE);
1424 		bus_dmamap_unload(txq->ift_desc_tag,
1425 				  txq->ift_sds.ifsd_map[i]);
1426 	}
1427 	m_free(*mp);
1428 	DBG_COUNTER_INC(tx_frees);
1429 	*mp = NULL;
1430 }
1431 
1432 static int
1433 iflib_txq_setup(iflib_txq_t txq)
1434 {
1435 	if_ctx_t ctx = txq->ift_ctx;
1436 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
1437 	iflib_dma_info_t di;
1438 	int i;
1439 
1440 	/* Set number of descriptors available */
1441 	txq->ift_qstatus = IFLIB_QUEUE_IDLE;
1442 
1443 	/* Reset indices */
1444 	txq->ift_cidx_processed = txq->ift_pidx = txq->ift_cidx = txq->ift_npending = 0;
1445 	txq->ift_size = scctx->isc_ntxd[txq->ift_br_offset];
1446 
1447 	for (i = 0, di = txq->ift_ifdi; i < ctx->ifc_nhwtxqs; i++, di++)
1448 		bzero((void *)di->idi_vaddr, di->idi_size);
1449 
1450 	IFDI_TXQ_SETUP(ctx, txq->ift_id);
1451 	for (i = 0, di = txq->ift_ifdi; i < ctx->ifc_nhwtxqs; i++, di++)
1452 		bus_dmamap_sync(di->idi_tag, di->idi_map,
1453 						BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1454 	return (0);
1455 }
1456 
1457 /*********************************************************************
1458  *
1459  *  Allocate memory for rx_buffer structures. Since we use one
1460  *  rx_buffer per received packet, the maximum number of rx_buffer's
1461  *  that we'll need is equal to the number of receive descriptors
1462  *  that we've allocated.
1463  *
1464  **********************************************************************/
1465 static int
1466 iflib_rxsd_alloc(iflib_rxq_t rxq)
1467 {
1468 	if_ctx_t ctx = rxq->ifr_ctx;
1469 	if_shared_ctx_t sctx = ctx->ifc_sctx;
1470 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
1471 	device_t dev = ctx->ifc_dev;
1472 	iflib_fl_t fl;
1473 	int			err;
1474 
1475 	MPASS(scctx->isc_nrxd[0] > 0);
1476 	MPASS(scctx->isc_nrxd[rxq->ifr_fl_offset] > 0);
1477 
1478 	fl = rxq->ifr_fl;
1479 	for (int i = 0; i <  rxq->ifr_nfl; i++, fl++) {
1480 		fl->ifl_size = scctx->isc_nrxd[rxq->ifr_fl_offset]; /* this isn't necessarily the same */
1481 		err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
1482 					 1, 0,			/* alignment, bounds */
1483 					 BUS_SPACE_MAXADDR,	/* lowaddr */
1484 					 BUS_SPACE_MAXADDR,	/* highaddr */
1485 					 NULL, NULL,		/* filter, filterarg */
1486 					 sctx->isc_rx_maxsize,	/* maxsize */
1487 					 sctx->isc_rx_nsegments,	/* nsegments */
1488 					 sctx->isc_rx_maxsegsize,	/* maxsegsize */
1489 					 0,			/* flags */
1490 					 NULL,			/* lockfunc */
1491 					 NULL,			/* lockarg */
1492 					 &fl->ifl_desc_tag);
1493 		if (err) {
1494 			device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
1495 				__func__, err);
1496 			goto fail;
1497 		}
1498 		if (!(fl->ifl_sds.ifsd_flags =
1499 		      (uint8_t *) malloc(sizeof(uint8_t) *
1500 					 scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
1501 			device_printf(dev, "Unable to allocate tx_buffer memory\n");
1502 			err = ENOMEM;
1503 			goto fail;
1504 		}
1505 		if (!(fl->ifl_sds.ifsd_m =
1506 		      (struct mbuf **) malloc(sizeof(struct mbuf *) *
1507 					      scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
1508 			device_printf(dev, "Unable to allocate tx_buffer memory\n");
1509 			err = ENOMEM;
1510 			goto fail;
1511 		}
1512 		if (!(fl->ifl_sds.ifsd_cl =
1513 		      (caddr_t *) malloc(sizeof(caddr_t) *
1514 					      scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
1515 			device_printf(dev, "Unable to allocate tx_buffer memory\n");
1516 			err = ENOMEM;
1517 			goto fail;
1518 		}
1519 
1520 		/* Create the descriptor buffer dma maps */
1521 #if defined(ACPI_DMAR) || (!(defined(__i386__) && !defined(__amd64__)))
1522 		if ((ctx->ifc_flags & IFC_DMAR) == 0)
1523 			continue;
1524 
1525 		if (!(fl->ifl_sds.ifsd_map =
1526 		      (bus_dmamap_t *) malloc(sizeof(bus_dmamap_t) * scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
1527 			device_printf(dev, "Unable to allocate tx_buffer map memory\n");
1528 			err = ENOMEM;
1529 			goto fail;
1530 		}
1531 
1532 		for (int i = 0; i < scctx->isc_nrxd[rxq->ifr_fl_offset]; i++) {
1533 			err = bus_dmamap_create(fl->ifl_desc_tag, 0, &fl->ifl_sds.ifsd_map[i]);
1534 			if (err != 0) {
1535 				device_printf(dev, "Unable to create TX DMA map\n");
1536 				goto fail;
1537 			}
1538 		}
1539 #endif
1540 	}
1541 	return (0);
1542 
1543 fail:
1544 	iflib_rx_structures_free(ctx);
1545 	return (err);
1546 }
1547 
1548 
1549 /*
1550  * Internal service routines
1551  */
1552 
1553 struct rxq_refill_cb_arg {
1554 	int               error;
1555 	bus_dma_segment_t seg;
1556 	int               nseg;
1557 };
1558 
1559 static void
1560 _rxq_refill_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
1561 {
1562 	struct rxq_refill_cb_arg *cb_arg = arg;
1563 
1564 	cb_arg->error = error;
1565 	cb_arg->seg = segs[0];
1566 	cb_arg->nseg = nseg;
1567 }
1568 
1569 
1570 #ifdef ACPI_DMAR
1571 #define IS_DMAR(ctx) (ctx->ifc_flags & IFC_DMAR)
1572 #else
1573 #define IS_DMAR(ctx) (0)
1574 #endif
1575 
1576 /**
1577  *	rxq_refill - refill an rxq  free-buffer list
1578  *	@ctx: the iflib context
1579  *	@rxq: the free-list to refill
1580  *	@n: the number of new buffers to allocate
1581  *
1582  *	(Re)populate an rxq free-buffer list with up to @n new packet buffers.
1583  *	The caller must assure that @n does not exceed the queue's capacity.
1584  */
1585 static void
1586 _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count)
1587 {
1588 	struct mbuf *m;
1589 	int idx, pidx = fl->ifl_pidx;
1590 	caddr_t cl, *sd_cl;
1591 	struct mbuf **sd_m;
1592 	uint8_t *sd_flags;
1593 	bus_dmamap_t *sd_map;
1594 	int n, i = 0;
1595 	uint64_t bus_addr;
1596 	int err;
1597 
1598 	sd_m = fl->ifl_sds.ifsd_m;
1599 	sd_map = fl->ifl_sds.ifsd_map;
1600 	sd_cl = fl->ifl_sds.ifsd_cl;
1601 	sd_flags = fl->ifl_sds.ifsd_flags;
1602 	idx = pidx;
1603 
1604 	n  = count;
1605 	MPASS(n > 0);
1606 	MPASS(fl->ifl_credits + n <= fl->ifl_size);
1607 
1608 	if (pidx < fl->ifl_cidx)
1609 		MPASS(pidx + n <= fl->ifl_cidx);
1610 	if (pidx == fl->ifl_cidx && (fl->ifl_credits < fl->ifl_size))
1611 		MPASS(fl->ifl_gen == 0);
1612 	if (pidx > fl->ifl_cidx)
1613 		MPASS(n <= fl->ifl_size - pidx + fl->ifl_cidx);
1614 
1615 	DBG_COUNTER_INC(fl_refills);
1616 	if (n > 8)
1617 		DBG_COUNTER_INC(fl_refills_large);
1618 
1619 	while (n--) {
1620 		/*
1621 		 * We allocate an uninitialized mbuf + cluster, mbuf is
1622 		 * initialized after rx.
1623 		 *
1624 		 * If the cluster is still set then we know a minimum sized packet was received
1625 		 */
1626 		if ((cl = sd_cl[idx]) == NULL) {
1627 			if ((cl = sd_cl[idx] = m_cljget(NULL, M_NOWAIT, fl->ifl_buf_size)) == NULL)
1628 				break;
1629 #if MEMORY_LOGGING
1630 			fl->ifl_cl_enqueued++;
1631 #endif
1632 		}
1633 		if ((m = m_gethdr(M_NOWAIT, MT_NOINIT)) == NULL) {
1634 			break;
1635 		}
1636 #if MEMORY_LOGGING
1637 		fl->ifl_m_enqueued++;
1638 #endif
1639 
1640 		DBG_COUNTER_INC(rx_allocs);
1641 #ifdef notyet
1642 		if ((sd_flags[pidx] & RX_SW_DESC_MAP_CREATED) == 0) {
1643 			int err;
1644 
1645 			if ((err = bus_dmamap_create(fl->ifl_ifdi->idi_tag, 0, &sd_map[idx]))) {
1646 				log(LOG_WARNING, "bus_dmamap_create failed %d\n", err);
1647 				uma_zfree(fl->ifl_zone, cl);
1648 				n = 0;
1649 				goto done;
1650 			}
1651 			sd_flags[idx] |= RX_SW_DESC_MAP_CREATED;
1652 		}
1653 #endif
1654 #if defined(__i386__) || defined(__amd64__)
1655 		if (!IS_DMAR(ctx)) {
1656 			bus_addr = pmap_kextract((vm_offset_t)cl);
1657 		} else
1658 #endif
1659 		{
1660 			struct rxq_refill_cb_arg cb_arg;
1661 			iflib_rxq_t q;
1662 
1663 			cb_arg.error = 0;
1664 			q = fl->ifl_rxq;
1665 			err = bus_dmamap_load(fl->ifl_desc_tag, sd_map[idx],
1666 		         cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg, 0);
1667 
1668 			if (err != 0 || cb_arg.error) {
1669 				/*
1670 				 * !zone_pack ?
1671 				 */
1672 				if (fl->ifl_zone == zone_pack)
1673 					uma_zfree(fl->ifl_zone, cl);
1674 				m_free(m);
1675 				n = 0;
1676 				goto done;
1677 			}
1678 			bus_addr = cb_arg.seg.ds_addr;
1679 		}
1680 		sd_flags[idx] |= RX_SW_DESC_INUSE;
1681 
1682 		MPASS(sd_m[idx] == NULL);
1683 		sd_cl[idx] = cl;
1684 		sd_m[idx] = m;
1685 		fl->ifl_bus_addrs[i] = bus_addr;
1686 		fl->ifl_vm_addrs[i] = cl;
1687 		fl->ifl_credits++;
1688 		i++;
1689 		MPASS(fl->ifl_credits <= fl->ifl_size);
1690 		if (++idx == fl->ifl_size) {
1691 			fl->ifl_gen = 1;
1692 			idx = 0;
1693 		}
1694 		if (n == 0 || i == IFLIB_MAX_RX_REFRESH) {
1695 			ctx->isc_rxd_refill(ctx->ifc_softc, fl->ifl_rxq->ifr_id, fl->ifl_id, pidx,
1696 								 fl->ifl_bus_addrs, fl->ifl_vm_addrs, i, fl->ifl_buf_size);
1697 			i = 0;
1698 			pidx = idx;
1699 		}
1700 		fl->ifl_pidx = idx;
1701 
1702 	}
1703 done:
1704 	DBG_COUNTER_INC(rxd_flush);
1705 	if (fl->ifl_pidx == 0)
1706 		pidx = fl->ifl_size - 1;
1707 	else
1708 		pidx = fl->ifl_pidx - 1;
1709 	ctx->isc_rxd_flush(ctx->ifc_softc, fl->ifl_rxq->ifr_id, fl->ifl_id, pidx);
1710 }
1711 
1712 static __inline void
1713 __iflib_fl_refill_lt(if_ctx_t ctx, iflib_fl_t fl, int max)
1714 {
1715 	/* we avoid allowing pidx to catch up with cidx as it confuses ixl */
1716 	int32_t reclaimable = fl->ifl_size - fl->ifl_credits - 1;
1717 #ifdef INVARIANTS
1718 	int32_t delta = fl->ifl_size - get_inuse(fl->ifl_size, fl->ifl_cidx, fl->ifl_pidx, fl->ifl_gen) - 1;
1719 #endif
1720 
1721 	MPASS(fl->ifl_credits <= fl->ifl_size);
1722 	MPASS(reclaimable == delta);
1723 
1724 	if (reclaimable > 0)
1725 		_iflib_fl_refill(ctx, fl, min(max, reclaimable));
1726 }
1727 
1728 static void
1729 iflib_fl_bufs_free(iflib_fl_t fl)
1730 {
1731 	iflib_dma_info_t idi = fl->ifl_ifdi;
1732 	uint32_t i;
1733 
1734 	for (i = 0; i < fl->ifl_size; i++) {
1735 		struct mbuf **sd_m = &fl->ifl_sds.ifsd_m[i];
1736 		uint8_t *sd_flags = &fl->ifl_sds.ifsd_flags[i];
1737 		caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i];
1738 
1739 		if (*sd_flags & RX_SW_DESC_INUSE) {
1740 			if (fl->ifl_sds.ifsd_map != NULL) {
1741 				bus_dmamap_t sd_map = fl->ifl_sds.ifsd_map[i];
1742 				bus_dmamap_unload(fl->ifl_desc_tag, sd_map);
1743 				bus_dmamap_destroy(fl->ifl_desc_tag, sd_map);
1744 			}
1745 			if (*sd_m != NULL) {
1746 				m_init(*sd_m, M_NOWAIT, MT_DATA, 0);
1747 				uma_zfree(zone_mbuf, *sd_m);
1748 			}
1749 			if (*sd_cl != NULL)
1750 				uma_zfree(fl->ifl_zone, *sd_cl);
1751 			*sd_flags = 0;
1752 		} else {
1753 			MPASS(*sd_cl == NULL);
1754 			MPASS(*sd_m == NULL);
1755 		}
1756 #if MEMORY_LOGGING
1757 		fl->ifl_m_dequeued++;
1758 		fl->ifl_cl_dequeued++;
1759 #endif
1760 		*sd_cl = NULL;
1761 		*sd_m = NULL;
1762 	}
1763 	/*
1764 	 * Reset free list values
1765 	 */
1766 	fl->ifl_credits = fl->ifl_cidx = fl->ifl_pidx = fl->ifl_gen = 0;;
1767 	bzero(idi->idi_vaddr, idi->idi_size);
1768 }
1769 
1770 /*********************************************************************
1771  *
1772  *  Initialize a receive ring and its buffers.
1773  *
1774  **********************************************************************/
1775 static int
1776 iflib_fl_setup(iflib_fl_t fl)
1777 {
1778 	iflib_rxq_t rxq = fl->ifl_rxq;
1779 	if_ctx_t ctx = rxq->ifr_ctx;
1780 	if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
1781 
1782 	/*
1783 	** Free current RX buffer structs and their mbufs
1784 	*/
1785 	iflib_fl_bufs_free(fl);
1786 	/* Now replenish the mbufs */
1787 	MPASS(fl->ifl_credits == 0);
1788 	/*
1789 	 * XXX don't set the max_frame_size to larger
1790 	 * than the hardware can handle
1791 	 */
1792 	if (sctx->isc_max_frame_size <= 2048)
1793 		fl->ifl_buf_size = MCLBYTES;
1794 	else if (sctx->isc_max_frame_size <= 4096)
1795 		fl->ifl_buf_size = MJUMPAGESIZE;
1796 	else if (sctx->isc_max_frame_size <= 9216)
1797 		fl->ifl_buf_size = MJUM9BYTES;
1798 	else
1799 		fl->ifl_buf_size = MJUM16BYTES;
1800 	if (fl->ifl_buf_size > ctx->ifc_max_fl_buf_size)
1801 		ctx->ifc_max_fl_buf_size = fl->ifl_buf_size;
1802 	fl->ifl_cltype = m_gettype(fl->ifl_buf_size);
1803 	fl->ifl_zone = m_getzone(fl->ifl_buf_size);
1804 
1805 
1806 	/* avoid pre-allocating zillions of clusters to an idle card
1807 	 * potentially speeding up attach
1808 	 */
1809 	_iflib_fl_refill(ctx, fl, min(128, fl->ifl_size));
1810 	MPASS(min(128, fl->ifl_size) == fl->ifl_credits);
1811 	if (min(128, fl->ifl_size) != fl->ifl_credits)
1812 		return (ENOBUFS);
1813 	/*
1814 	 * handle failure
1815 	 */
1816 	MPASS(rxq != NULL);
1817 	MPASS(fl->ifl_ifdi != NULL);
1818 	bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
1819 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1820 	return (0);
1821 }
1822 
1823 /*********************************************************************
1824  *
1825  *  Free receive ring data structures
1826  *
1827  **********************************************************************/
1828 static void
1829 iflib_rx_sds_free(iflib_rxq_t rxq)
1830 {
1831 	iflib_fl_t fl;
1832 	int i;
1833 
1834 	if (rxq->ifr_fl != NULL) {
1835 		for (i = 0; i < rxq->ifr_nfl; i++) {
1836 			fl = &rxq->ifr_fl[i];
1837 			if (fl->ifl_desc_tag != NULL) {
1838 				bus_dma_tag_destroy(fl->ifl_desc_tag);
1839 				fl->ifl_desc_tag = NULL;
1840 			}
1841 			free(fl->ifl_sds.ifsd_m, M_IFLIB);
1842 			free(fl->ifl_sds.ifsd_cl, M_IFLIB);
1843 			/* XXX destroy maps first */
1844 			free(fl->ifl_sds.ifsd_map, M_IFLIB);
1845 			fl->ifl_sds.ifsd_m = NULL;
1846 			fl->ifl_sds.ifsd_cl = NULL;
1847 			fl->ifl_sds.ifsd_map = NULL;
1848 		}
1849 		free(rxq->ifr_fl, M_IFLIB);
1850 		rxq->ifr_fl = NULL;
1851 		rxq->ifr_cq_gen = rxq->ifr_cq_cidx = rxq->ifr_cq_pidx = 0;
1852 	}
1853 }
1854 
1855 /*
1856  * MI independent logic
1857  *
1858  */
1859 static void
1860 iflib_timer(void *arg)
1861 {
1862 	iflib_txq_t txq = arg;
1863 	if_ctx_t ctx = txq->ift_ctx;
1864 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
1865 
1866 	if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
1867 		return;
1868 	/*
1869 	** Check on the state of the TX queue(s), this
1870 	** can be done without the lock because its RO
1871 	** and the HUNG state will be static if set.
1872 	*/
1873 	IFDI_TIMER(ctx, txq->ift_id);
1874 	if ((txq->ift_qstatus == IFLIB_QUEUE_HUNG) &&
1875 		(ctx->ifc_pause_frames == 0))
1876 		goto hung;
1877 
1878 	if (TXQ_AVAIL(txq) <= 2*scctx->isc_tx_nsegments ||
1879 	    ifmp_ring_is_stalled(txq->ift_br[0]))
1880 		GROUPTASK_ENQUEUE(&txq->ift_task);
1881 
1882 	ctx->ifc_pause_frames = 0;
1883 	if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)
1884 		callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, txq->ift_timer.c_cpu);
1885 	return;
1886 hung:
1887 	CTX_LOCK(ctx);
1888 	if_setdrvflagbits(ctx->ifc_ifp, 0, IFF_DRV_RUNNING);
1889 	device_printf(ctx->ifc_dev,  "TX(%d) desc avail = %d, pidx = %d\n",
1890 				  txq->ift_id, TXQ_AVAIL(txq), txq->ift_pidx);
1891 
1892 	IFDI_WATCHDOG_RESET(ctx);
1893 	ctx->ifc_watchdog_events++;
1894 	ctx->ifc_pause_frames = 0;
1895 
1896 	iflib_init_locked(ctx);
1897 	CTX_UNLOCK(ctx);
1898 }
1899 
1900 static void
1901 iflib_init_locked(if_ctx_t ctx)
1902 {
1903 	if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
1904 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
1905 	if_t ifp = ctx->ifc_ifp;
1906 	iflib_fl_t fl;
1907 	iflib_txq_t txq;
1908 	iflib_rxq_t rxq;
1909 	int i, j, tx_ip_csum_flags, tx_ip6_csum_flags;
1910 
1911 
1912 	if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
1913 	IFDI_INTR_DISABLE(ctx);
1914 
1915 	tx_ip_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP);
1916 	tx_ip6_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_IP6_SCTP);
1917 	/* Set hardware offload abilities */
1918 	if_clearhwassist(ifp);
1919 	if (if_getcapenable(ifp) & IFCAP_TXCSUM)
1920 		if_sethwassistbits(ifp, tx_ip_csum_flags, 0);
1921 	if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6)
1922 		if_sethwassistbits(ifp,  tx_ip6_csum_flags, 0);
1923 	if (if_getcapenable(ifp) & IFCAP_TSO4)
1924 		if_sethwassistbits(ifp, CSUM_IP_TSO, 0);
1925 	if (if_getcapenable(ifp) & IFCAP_TSO6)
1926 		if_sethwassistbits(ifp, CSUM_IP6_TSO, 0);
1927 
1928 	for (i = 0, txq = ctx->ifc_txqs; i < sctx->isc_ntxqsets; i++, txq++) {
1929 		CALLOUT_LOCK(txq);
1930 		callout_stop(&txq->ift_timer);
1931 		callout_stop(&txq->ift_db_check);
1932 		CALLOUT_UNLOCK(txq);
1933 		iflib_netmap_txq_init(ctx, txq);
1934 	}
1935 	for (i = 0, rxq = ctx->ifc_rxqs; i < sctx->isc_nrxqsets; i++, rxq++) {
1936 		iflib_netmap_rxq_init(ctx, rxq);
1937 	}
1938 #ifdef INVARIANTS
1939 	i = if_getdrvflags(ifp);
1940 #endif
1941 	IFDI_INIT(ctx);
1942 	MPASS(if_getdrvflags(ifp) == i);
1943 	for (i = 0, rxq = ctx->ifc_rxqs; i < sctx->isc_nrxqsets; i++, rxq++) {
1944 		for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) {
1945 			if (iflib_fl_setup(fl)) {
1946 				device_printf(ctx->ifc_dev, "freelist setup failed - check cluster settings\n");
1947 				goto done;
1948 			}
1949 		}
1950 	}
1951 	done:
1952 	if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
1953 	IFDI_INTR_ENABLE(ctx);
1954 	txq = ctx->ifc_txqs;
1955 	for (i = 0; i < sctx->isc_ntxqsets; i++, txq++)
1956 		callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq,
1957 			txq->ift_timer.c_cpu);
1958 }
1959 
1960 static int
1961 iflib_media_change(if_t ifp)
1962 {
1963 	if_ctx_t ctx = if_getsoftc(ifp);
1964 	int err;
1965 
1966 	CTX_LOCK(ctx);
1967 	if ((err = IFDI_MEDIA_CHANGE(ctx)) == 0)
1968 		iflib_init_locked(ctx);
1969 	CTX_UNLOCK(ctx);
1970 	return (err);
1971 }
1972 
1973 static void
1974 iflib_media_status(if_t ifp, struct ifmediareq *ifmr)
1975 {
1976 	if_ctx_t ctx = if_getsoftc(ifp);
1977 
1978 	CTX_LOCK(ctx);
1979 	IFDI_UPDATE_ADMIN_STATUS(ctx);
1980 	IFDI_MEDIA_STATUS(ctx, ifmr);
1981 	CTX_UNLOCK(ctx);
1982 }
1983 
1984 static void
1985 iflib_stop(if_ctx_t ctx)
1986 {
1987 	iflib_txq_t txq = ctx->ifc_txqs;
1988 	iflib_rxq_t rxq = ctx->ifc_rxqs;
1989 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
1990 	iflib_dma_info_t di;
1991 	iflib_fl_t fl;
1992 	int i, j;
1993 
1994 	/* Tell the stack that the interface is no longer active */
1995 	if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
1996 
1997 	IFDI_INTR_DISABLE(ctx);
1998 	DELAY(100000);
1999 	IFDI_STOP(ctx);
2000 	DELAY(100000);
2001 
2002 	iflib_debug_reset();
2003 	/* Wait for current tx queue users to exit to disarm watchdog timer. */
2004 	for (i = 0; i < scctx->isc_ntxqsets; i++, txq++) {
2005 		/* make sure all transmitters have completed before proceeding XXX */
2006 
2007 		/* clean any enqueued buffers */
2008 		iflib_ifmp_purge(txq);
2009 		/* Free any existing tx buffers. */
2010 		for (j = 0; j < txq->ift_size; j++) {
2011 			iflib_txsd_free(ctx, txq, j);
2012 		}
2013 		txq->ift_processed = txq->ift_cleaned = txq->ift_cidx_processed = 0;
2014 		txq->ift_in_use = txq->ift_gen = txq->ift_cidx = txq->ift_pidx = txq->ift_no_desc_avail = 0;
2015 		txq->ift_closed = txq->ift_mbuf_defrag = txq->ift_mbuf_defrag_failed = 0;
2016 		txq->ift_no_tx_dma_setup = txq->ift_txd_encap_efbig = txq->ift_map_failed = 0;
2017 		txq->ift_pullups = 0;
2018 		ifmp_ring_reset_stats(txq->ift_br[0]);
2019 		for (j = 0, di = txq->ift_ifdi; j < ctx->ifc_nhwtxqs; j++, di++)
2020 			bzero((void *)di->idi_vaddr, di->idi_size);
2021 	}
2022 	for (i = 0; i < scctx->isc_nrxqsets; i++, rxq++) {
2023 		/* make sure all transmitters have completed before proceeding XXX */
2024 
2025 		for (j = 0, di = txq->ift_ifdi; j < ctx->ifc_nhwrxqs; j++, di++)
2026 			bzero((void *)di->idi_vaddr, di->idi_size);
2027 		/* also resets the free lists pidx/cidx */
2028 		for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)
2029 			iflib_fl_bufs_free(fl);
2030 	}
2031 }
2032 
2033 static inline void
2034 prefetch_pkts(iflib_fl_t fl, int cidx)
2035 {
2036 	int nextptr;
2037 	int nrxd = fl->ifl_size;
2038 
2039 	nextptr = (cidx + CACHE_PTR_INCREMENT) & (nrxd-1);
2040 	prefetch(&fl->ifl_sds.ifsd_m[nextptr]);
2041 	prefetch(&fl->ifl_sds.ifsd_cl[nextptr]);
2042 	prefetch(fl->ifl_sds.ifsd_m[(cidx + 1) & (nrxd-1)]);
2043 	prefetch(fl->ifl_sds.ifsd_m[(cidx + 2) & (nrxd-1)]);
2044 	prefetch(fl->ifl_sds.ifsd_m[(cidx + 3) & (nrxd-1)]);
2045 	prefetch(fl->ifl_sds.ifsd_m[(cidx + 4) & (nrxd-1)]);
2046 	prefetch(fl->ifl_sds.ifsd_cl[(cidx + 1) & (nrxd-1)]);
2047 	prefetch(fl->ifl_sds.ifsd_cl[(cidx + 2) & (nrxd-1)]);
2048 	prefetch(fl->ifl_sds.ifsd_cl[(cidx + 3) & (nrxd-1)]);
2049 	prefetch(fl->ifl_sds.ifsd_cl[(cidx + 4) & (nrxd-1)]);
2050 }
2051 
2052 static void
2053 rxd_frag_to_sd(iflib_rxq_t rxq, if_rxd_frag_t irf, int *cltype, int unload, iflib_fl_t *pfl, int *pcidx)
2054 {
2055 	int flid, cidx;
2056 	bus_dmamap_t map;
2057 	iflib_fl_t fl;
2058 	iflib_dma_info_t di;
2059 	int next;
2060 
2061 	flid = irf->irf_flid;
2062 	cidx = irf->irf_idx;
2063 	fl = &rxq->ifr_fl[flid];
2064 	fl->ifl_credits--;
2065 #if MEMORY_LOGGING
2066 	fl->ifl_m_dequeued++;
2067 	if (cltype)
2068 		fl->ifl_cl_dequeued++;
2069 #endif
2070 	prefetch_pkts(fl, cidx);
2071 	if (fl->ifl_sds.ifsd_map != NULL) {
2072 		next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size-1);
2073 		prefetch(&fl->ifl_sds.ifsd_map[next]);
2074 		map = fl->ifl_sds.ifsd_map[cidx];
2075 		di = fl->ifl_ifdi;
2076 		next = (cidx + CACHE_LINE_SIZE) & (fl->ifl_size-1);
2077 		prefetch(&fl->ifl_sds.ifsd_flags[next]);
2078 		bus_dmamap_sync(di->idi_tag, di->idi_map,
2079 				BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2080 
2081 	/* not valid assert if bxe really does SGE from non-contiguous elements */
2082 		MPASS(fl->ifl_cidx == cidx);
2083 		if (unload)
2084 			bus_dmamap_unload(fl->ifl_desc_tag, map);
2085 	}
2086 	if (__predict_false(++fl->ifl_cidx == fl->ifl_size)) {
2087 		fl->ifl_cidx = 0;
2088 		fl->ifl_gen = 0;
2089 	}
2090 	/* YES ick */
2091 	if (cltype)
2092 		*cltype = fl->ifl_cltype;
2093 	*pfl = fl;
2094 	*pcidx = cidx;
2095 }
2096 
2097 static struct mbuf *
2098 assemble_segments(iflib_rxq_t rxq, if_rxd_info_t ri)
2099 {
2100 	int i, padlen , flags, cltype;
2101 	struct mbuf *m, *mh, *mt, *sd_m;
2102 	iflib_fl_t fl;
2103 	int cidx;
2104 	caddr_t cl, sd_cl;
2105 
2106 	i = 0;
2107 	mh = NULL;
2108 	do {
2109 		rxd_frag_to_sd(rxq, &ri->iri_frags[i], &cltype, TRUE, &fl, &cidx);
2110 		sd_m = fl->ifl_sds.ifsd_m[cidx];
2111 		sd_cl = fl->ifl_sds.ifsd_cl[cidx];
2112 
2113 		MPASS(sd_cl != NULL);
2114 		MPASS(sd_m != NULL);
2115 
2116 		/* Don't include zero-length frags */
2117 		if (ri->iri_frags[i].irf_len == 0) {
2118 			/* XXX we can save the cluster here, but not the mbuf */
2119 			m_init(sd_m, M_NOWAIT, MT_DATA, 0);
2120 			m_free(sd_m);
2121 			fl->ifl_sds.ifsd_m[cidx] = NULL;
2122 			continue;
2123 		}
2124 		m = sd_m;
2125 		if (mh == NULL) {
2126 			flags = M_PKTHDR|M_EXT;
2127 			mh = mt = m;
2128 			padlen = ri->iri_pad;
2129 		} else {
2130 			flags = M_EXT;
2131 			mt->m_next = m;
2132 			mt = m;
2133 			/* assuming padding is only on the first fragment */
2134 			padlen = 0;
2135 		}
2136 		fl->ifl_sds.ifsd_m[cidx] = NULL;
2137 		cl = fl->ifl_sds.ifsd_cl[cidx];
2138 		fl->ifl_sds.ifsd_cl[cidx] = NULL;
2139 
2140 		/* Can these two be made one ? */
2141 		m_init(m, M_NOWAIT, MT_DATA, flags);
2142 		m_cljset(m, cl, cltype);
2143 		/*
2144 		 * These must follow m_init and m_cljset
2145 		 */
2146 		m->m_data += padlen;
2147 		ri->iri_len -= padlen;
2148 		m->m_len = ri->iri_frags[i].irf_len;
2149 	} while (++i < ri->iri_nfrags);
2150 
2151 	return (mh);
2152 }
2153 
2154 /*
2155  * Process one software descriptor
2156  */
2157 static struct mbuf *
2158 iflib_rxd_pkt_get(iflib_rxq_t rxq, if_rxd_info_t ri)
2159 {
2160 	struct mbuf *m;
2161 	iflib_fl_t fl;
2162 	caddr_t sd_cl;
2163 	int cidx;
2164 
2165 	/* should I merge this back in now that the two paths are basically duplicated? */
2166 	if (ri->iri_nfrags == 1 &&
2167 	    ri->iri_frags[0].irf_len <= IFLIB_RX_COPY_THRESH) {
2168 		rxd_frag_to_sd(rxq, &ri->iri_frags[0], NULL, FALSE, &fl, &cidx);
2169 		m = fl->ifl_sds.ifsd_m[cidx];
2170 		fl->ifl_sds.ifsd_m[cidx] = NULL;
2171 		sd_cl = fl->ifl_sds.ifsd_cl[cidx];
2172 		m_init(m, M_NOWAIT, MT_DATA, M_PKTHDR);
2173 		memcpy(m->m_data, sd_cl, ri->iri_len);
2174 		m->m_len = ri->iri_frags[0].irf_len;
2175        } else {
2176 		m = assemble_segments(rxq, ri);
2177 	}
2178 	m->m_pkthdr.len = ri->iri_len;
2179 	m->m_pkthdr.rcvif = ri->iri_ifp;
2180 	m->m_flags |= ri->iri_flags;
2181 	m->m_pkthdr.ether_vtag = ri->iri_vtag;
2182 	m->m_pkthdr.flowid = ri->iri_flowid;
2183 	M_HASHTYPE_SET(m, ri->iri_rsstype);
2184 	m->m_pkthdr.csum_flags = ri->iri_csum_flags;
2185 	m->m_pkthdr.csum_data = ri->iri_csum_data;
2186 	return (m);
2187 }
2188 
2189 static bool
2190 iflib_rxeof(iflib_rxq_t rxq, int budget)
2191 {
2192 	if_ctx_t ctx = rxq->ifr_ctx;
2193 	if_shared_ctx_t sctx = ctx->ifc_sctx;
2194 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
2195 	int avail, i;
2196 	uint16_t *cidxp;
2197 	struct if_rxd_info ri;
2198 	int err, budget_left, rx_bytes, rx_pkts;
2199 	iflib_fl_t fl;
2200 	struct ifnet *ifp;
2201 	int lro_enabled;
2202 	/*
2203 	 * XXX early demux data packets so that if_input processing only handles
2204 	 * acks in interrupt context
2205 	 */
2206 	struct mbuf *m, *mh, *mt;
2207 
2208 	if (netmap_rx_irq(ctx->ifc_ifp, rxq->ifr_id, &budget)) {
2209 		return (FALSE);
2210 	}
2211 
2212 	mh = mt = NULL;
2213 	MPASS(budget > 0);
2214 	rx_pkts	= rx_bytes = 0;
2215 	if (sctx->isc_flags & IFLIB_HAS_RXCQ)
2216 		cidxp = &rxq->ifr_cq_cidx;
2217 	else
2218 		cidxp = &rxq->ifr_fl[0].ifl_cidx;
2219 	if ((avail = iflib_rxd_avail(ctx, rxq, *cidxp, budget)) == 0) {
2220 		for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++)
2221 			__iflib_fl_refill_lt(ctx, fl, budget + 8);
2222 		DBG_COUNTER_INC(rx_unavail);
2223 		return (false);
2224 	}
2225 
2226 	for (budget_left = budget; (budget_left > 0) && (avail > 0); budget_left--, avail--) {
2227 		if (__predict_false(!CTX_ACTIVE(ctx))) {
2228 			DBG_COUNTER_INC(rx_ctx_inactive);
2229 			break;
2230 		}
2231 		/*
2232 		 * Reset client set fields to their default values
2233 		 */
2234 		bzero(&ri, sizeof(ri));
2235 		ri.iri_qsidx = rxq->ifr_id;
2236 		ri.iri_cidx = *cidxp;
2237 		ri.iri_ifp = ctx->ifc_ifp;
2238 		ri.iri_frags = rxq->ifr_frags;
2239 		err = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri);
2240 
2241 		/* in lieu of handling correctly - make sure it isn't being unhandled */
2242 		MPASS(err == 0);
2243 		if (sctx->isc_flags & IFLIB_HAS_RXCQ) {
2244 			*cidxp = ri.iri_cidx;
2245 			/* Update our consumer index */
2246 			while (rxq->ifr_cq_cidx >= scctx->isc_nrxd[0]) {
2247 				rxq->ifr_cq_cidx -= scctx->isc_nrxd[0];
2248 				rxq->ifr_cq_gen = 0;
2249 			}
2250 			/* was this only a completion queue message? */
2251 			if (__predict_false(ri.iri_nfrags == 0))
2252 				continue;
2253 		}
2254 		MPASS(ri.iri_nfrags != 0);
2255 		MPASS(ri.iri_len != 0);
2256 
2257 		/* will advance the cidx on the corresponding free lists */
2258 		m = iflib_rxd_pkt_get(rxq, &ri);
2259 		if (avail == 0 && budget_left)
2260 			avail = iflib_rxd_avail(ctx, rxq, *cidxp, budget_left);
2261 
2262 		if (__predict_false(m == NULL)) {
2263 			DBG_COUNTER_INC(rx_mbuf_null);
2264 			continue;
2265 		}
2266 		/* imm_pkt: -- cxgb */
2267 		if (mh == NULL)
2268 			mh = mt = m;
2269 		else {
2270 			mt->m_nextpkt = m;
2271 			mt = m;
2272 		}
2273 	}
2274 	/* make sure that we can refill faster than drain */
2275 	for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++)
2276 		__iflib_fl_refill_lt(ctx, fl, budget + 8);
2277 
2278 	ifp = ctx->ifc_ifp;
2279 	lro_enabled = (if_getcapenable(ifp) & IFCAP_LRO);
2280 	while (mh != NULL) {
2281 		m = mh;
2282 		mh = mh->m_nextpkt;
2283 		m->m_nextpkt = NULL;
2284 		rx_bytes += m->m_pkthdr.len;
2285 		rx_pkts++;
2286 #if defined(INET6) || defined(INET)
2287 		if (lro_enabled && tcp_lro_rx(&rxq->ifr_lc, m, 0) == 0)
2288 			continue;
2289 #endif
2290 		DBG_COUNTER_INC(rx_if_input);
2291 		ifp->if_input(ifp, m);
2292 	}
2293 
2294 	if_inc_counter(ifp, IFCOUNTER_IBYTES, rx_bytes);
2295 	if_inc_counter(ifp, IFCOUNTER_IPACKETS, rx_pkts);
2296 
2297 	/*
2298 	 * Flush any outstanding LRO work
2299 	 */
2300 #if defined(INET6) || defined(INET)
2301 	tcp_lro_flush_all(&rxq->ifr_lc);
2302 #endif
2303 	if (avail)
2304 		return true;
2305 	return (iflib_rxd_avail(ctx, rxq, *cidxp, 1));
2306 }
2307 
2308 #define M_CSUM_FLAGS(m) ((m)->m_pkthdr.csum_flags)
2309 #define M_HAS_VLANTAG(m) (m->m_flags & M_VLANTAG)
2310 #define TXQ_MAX_DB_DEFERRED(size) (size >> 5)
2311 #define TXQ_MAX_DB_CONSUMED(size) (size >> 4)
2312 
2313 static __inline void
2314 iflib_txd_db_check(if_ctx_t ctx, iflib_txq_t txq, int ring)
2315 {
2316 	uint32_t dbval;
2317 
2318 	if (ring || txq->ift_db_pending >=
2319 	    TXQ_MAX_DB_DEFERRED(txq->ift_size)) {
2320 
2321 		/* the lock will only ever be contended in the !min_latency case */
2322 		if (!TXDB_TRYLOCK(txq))
2323 			return;
2324 		dbval = txq->ift_npending ? txq->ift_npending : txq->ift_pidx;
2325 		ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, dbval);
2326 		txq->ift_db_pending = txq->ift_npending = 0;
2327 		TXDB_UNLOCK(txq);
2328 	}
2329 }
2330 
2331 static void
2332 iflib_txd_deferred_db_check(void * arg)
2333 {
2334 	iflib_txq_t txq = arg;
2335 
2336 	/* simple non-zero boolean so use bitwise OR */
2337 	if ((txq->ift_db_pending | txq->ift_npending) &&
2338 	    txq->ift_db_pending >= txq->ift_db_pending_queued)
2339 		iflib_txd_db_check(txq->ift_ctx, txq, TRUE);
2340 	txq->ift_db_pending_queued = 0;
2341 	if (ifmp_ring_is_stalled(txq->ift_br[0]))
2342 		iflib_txq_check_drain(txq, 4);
2343 }
2344 
2345 #ifdef PKT_DEBUG
2346 static void
2347 print_pkt(if_pkt_info_t pi)
2348 {
2349 	printf("pi len:  %d qsidx: %d nsegs: %d ndescs: %d flags: %x pidx: %d\n",
2350 	       pi->ipi_len, pi->ipi_qsidx, pi->ipi_nsegs, pi->ipi_ndescs, pi->ipi_flags, pi->ipi_pidx);
2351 	printf("pi new_pidx: %d csum_flags: %lx tso_segsz: %d mflags: %x vtag: %d\n",
2352 	       pi->ipi_new_pidx, pi->ipi_csum_flags, pi->ipi_tso_segsz, pi->ipi_mflags, pi->ipi_vtag);
2353 	printf("pi etype: %d ehdrlen: %d ip_hlen: %d ipproto: %d\n",
2354 	       pi->ipi_etype, pi->ipi_ehdrlen, pi->ipi_ip_hlen, pi->ipi_ipproto);
2355 }
2356 #endif
2357 
2358 #define IS_TSO4(pi) ((pi)->ipi_csum_flags & CSUM_IP_TSO)
2359 #define IS_TSO6(pi) ((pi)->ipi_csum_flags & CSUM_IP6_TSO)
2360 
2361 static int
2362 iflib_parse_header(iflib_txq_t txq, if_pkt_info_t pi, struct mbuf **mp)
2363 {
2364 	if_shared_ctx_t sctx = txq->ift_ctx->ifc_sctx;
2365 	struct ether_vlan_header *eh;
2366 	struct mbuf *m, *n;
2367 
2368 	n = m = *mp;
2369 	if ((sctx->isc_flags & IFLIB_NEED_SCRATCH) &&
2370 	    M_WRITABLE(m) == 0) {
2371 		if ((m = m_dup(m, M_NOWAIT)) == NULL) {
2372 			return (ENOMEM);
2373 		} else {
2374 			m_freem(*mp);
2375 			n = *mp = m;
2376 		}
2377 	}
2378 
2379 	/*
2380 	 * Determine where frame payload starts.
2381 	 * Jump over vlan headers if already present,
2382 	 * helpful for QinQ too.
2383 	 */
2384 	if (__predict_false(m->m_len < sizeof(*eh))) {
2385 		txq->ift_pullups++;
2386 		if (__predict_false((m = m_pullup(m, sizeof(*eh))) == NULL))
2387 			return (ENOMEM);
2388 	}
2389 	eh = mtod(m, struct ether_vlan_header *);
2390 	if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
2391 		pi->ipi_etype = ntohs(eh->evl_proto);
2392 		pi->ipi_ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
2393 	} else {
2394 		pi->ipi_etype = ntohs(eh->evl_encap_proto);
2395 		pi->ipi_ehdrlen = ETHER_HDR_LEN;
2396 	}
2397 
2398 	switch (pi->ipi_etype) {
2399 #ifdef INET
2400 	case ETHERTYPE_IP:
2401 	{
2402 		struct ip *ip = NULL;
2403 		struct tcphdr *th = NULL;
2404 		int minthlen;
2405 
2406 		minthlen = min(m->m_pkthdr.len, pi->ipi_ehdrlen + sizeof(*ip) + sizeof(*th));
2407 		if (__predict_false(m->m_len < minthlen)) {
2408 			/*
2409 			 * if this code bloat is causing too much of a hit
2410 			 * move it to a separate function and mark it noinline
2411 			 */
2412 			if (m->m_len == pi->ipi_ehdrlen) {
2413 				n = m->m_next;
2414 				MPASS(n);
2415 				if (n->m_len >= sizeof(*ip))  {
2416 					ip = (struct ip *)n->m_data;
2417 					if (n->m_len >= (ip->ip_hl << 2) + sizeof(*th))
2418 						th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
2419 				} else {
2420 					txq->ift_pullups++;
2421 					if (__predict_false((m = m_pullup(m, minthlen)) == NULL))
2422 						return (ENOMEM);
2423 					ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen);
2424 				}
2425 			} else {
2426 				txq->ift_pullups++;
2427 				if (__predict_false((m = m_pullup(m, minthlen)) == NULL))
2428 					return (ENOMEM);
2429 				ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen);
2430 				if (m->m_len >= (ip->ip_hl << 2) + sizeof(*th))
2431 					th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
2432 			}
2433 		} else {
2434 			ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen);
2435 			if (m->m_len >= (ip->ip_hl << 2) + sizeof(*th))
2436 				th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
2437 		}
2438 		pi->ipi_ip_hlen = ip->ip_hl << 2;
2439 		pi->ipi_ipproto = ip->ip_p;
2440 		pi->ipi_flags |= IPI_TX_IPV4;
2441 
2442 		if (pi->ipi_csum_flags & CSUM_IP)
2443                        ip->ip_sum = 0;
2444 
2445 		if (pi->ipi_ipproto == IPPROTO_TCP) {
2446 			if (__predict_false(th == NULL)) {
2447 				txq->ift_pullups++;
2448 				if (__predict_false((m = m_pullup(m, (ip->ip_hl << 2) + sizeof(*th))) == NULL))
2449 					return (ENOMEM);
2450 				th = (struct tcphdr *)((caddr_t)ip + pi->ipi_ip_hlen);
2451 			}
2452 			pi->ipi_tcp_hflags = th->th_flags;
2453 			pi->ipi_tcp_hlen = th->th_off << 2;
2454 			pi->ipi_tcp_seq = th->th_seq;
2455 		}
2456 		if (IS_TSO4(pi)) {
2457 			if (__predict_false(ip->ip_p != IPPROTO_TCP))
2458 				return (ENXIO);
2459 			th->th_sum = in_pseudo(ip->ip_src.s_addr,
2460 					       ip->ip_dst.s_addr, htons(IPPROTO_TCP));
2461 			pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz;
2462 			if (sctx->isc_flags & IFLIB_TSO_INIT_IP) {
2463 				ip->ip_sum = 0;
2464 				ip->ip_len = htons(pi->ipi_ip_hlen + pi->ipi_tcp_hlen + pi->ipi_tso_segsz);
2465 			}
2466 		}
2467 		break;
2468 	}
2469 #endif
2470 #ifdef INET6
2471 	case ETHERTYPE_IPV6:
2472 	{
2473 		struct ip6_hdr *ip6 = (struct ip6_hdr *)(m->m_data + pi->ipi_ehdrlen);
2474 		struct tcphdr *th;
2475 		pi->ipi_ip_hlen = sizeof(struct ip6_hdr);
2476 
2477 		if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr))) {
2478 			if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr))) == NULL))
2479 				return (ENOMEM);
2480 		}
2481 		th = (struct tcphdr *)((caddr_t)ip6 + pi->ipi_ip_hlen);
2482 
2483 		/* XXX-BZ this will go badly in case of ext hdrs. */
2484 		pi->ipi_ipproto = ip6->ip6_nxt;
2485 		pi->ipi_flags |= IPI_TX_IPV6;
2486 
2487 		if (pi->ipi_ipproto == IPPROTO_TCP) {
2488 			if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) {
2489 				if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) == NULL))
2490 					return (ENOMEM);
2491 			}
2492 			pi->ipi_tcp_hflags = th->th_flags;
2493 			pi->ipi_tcp_hlen = th->th_off << 2;
2494 		}
2495 		if (IS_TSO6(pi)) {
2496 
2497 			if (__predict_false(ip6->ip6_nxt != IPPROTO_TCP))
2498 				return (ENXIO);
2499 			/*
2500 			 * The corresponding flag is set by the stack in the IPv4
2501 			 * TSO case, but not in IPv6 (at least in FreeBSD 10.2).
2502 			 * So, set it here because the rest of the flow requires it.
2503 			 */
2504 			pi->ipi_csum_flags |= CSUM_TCP_IPV6;
2505 			th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
2506 			pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz;
2507 		}
2508 		break;
2509 	}
2510 #endif
2511 	default:
2512 		pi->ipi_csum_flags &= ~CSUM_OFFLOAD;
2513 		pi->ipi_ip_hlen = 0;
2514 		break;
2515 	}
2516 	*mp = m;
2517 
2518 	return (0);
2519 }
2520 
2521 static  __noinline  struct mbuf *
2522 collapse_pkthdr(struct mbuf *m0)
2523 {
2524 	struct mbuf *m, *m_next, *tmp;
2525 
2526 	m = m0;
2527 	m_next = m->m_next;
2528 	while (m_next != NULL && m_next->m_len == 0) {
2529 		m = m_next;
2530 		m->m_next = NULL;
2531 		m_free(m);
2532 		m_next = m_next->m_next;
2533 	}
2534 	m = m0;
2535 	m->m_next = m_next;
2536 	if ((m_next->m_flags & M_EXT) == 0) {
2537 		m = m_defrag(m, M_NOWAIT);
2538 	} else {
2539 		tmp = m_next->m_next;
2540 		memcpy(m_next, m, MPKTHSIZE);
2541 		m = m_next;
2542 		m->m_next = tmp;
2543 	}
2544 	return (m);
2545 }
2546 
2547 /*
2548  * If dodgy hardware rejects the scatter gather chain we've handed it
2549  * we'll need to remove the mbuf chain from ifsg_m[] before we can add the
2550  * m_defrag'd mbufs
2551  */
2552 static __noinline struct mbuf *
2553 iflib_remove_mbuf(iflib_txq_t txq)
2554 {
2555 	int ntxd, i, pidx;
2556 	struct mbuf *m, *mh, **ifsd_m;
2557 
2558 	pidx = txq->ift_pidx;
2559 	ifsd_m = txq->ift_sds.ifsd_m;
2560 	ntxd = txq->ift_size;
2561 	mh = m = ifsd_m[pidx];
2562 	ifsd_m[pidx] = NULL;
2563 #if MEMORY_LOGGING
2564 	txq->ift_dequeued++;
2565 #endif
2566 	i = 1;
2567 
2568 	while (m) {
2569 		ifsd_m[(pidx + i) & (ntxd -1)] = NULL;
2570 #if MEMORY_LOGGING
2571 		txq->ift_dequeued++;
2572 #endif
2573 		m = m->m_next;
2574 		i++;
2575 	}
2576 	return (mh);
2577 }
2578 
2579 static int
2580 iflib_busdma_load_mbuf_sg(iflib_txq_t txq, bus_dma_tag_t tag, bus_dmamap_t map,
2581 			  struct mbuf **m0, bus_dma_segment_t *segs, int *nsegs,
2582 			  int max_segs, int flags)
2583 {
2584 	if_ctx_t ctx;
2585 	if_shared_ctx_t		sctx;
2586 	if_softc_ctx_t		scctx;
2587 	int i, next, pidx, mask, err, maxsegsz, ntxd, count;
2588 	struct mbuf *m, *tmp, **ifsd_m, **mp;
2589 
2590 	m = *m0;
2591 
2592 	/*
2593 	 * Please don't ever do this
2594 	 */
2595 	if (__predict_false(m->m_len == 0))
2596 		*m0 = m = collapse_pkthdr(m);
2597 
2598 	ctx = txq->ift_ctx;
2599 	sctx = ctx->ifc_sctx;
2600 	scctx = &ctx->ifc_softc_ctx;
2601 	ifsd_m = txq->ift_sds.ifsd_m;
2602 	ntxd = txq->ift_size;
2603 	pidx = txq->ift_pidx;
2604 	if (map != NULL) {
2605 		uint8_t *ifsd_flags = txq->ift_sds.ifsd_flags;
2606 
2607 		err = bus_dmamap_load_mbuf_sg(tag, map,
2608 					      *m0, segs, nsegs, BUS_DMA_NOWAIT);
2609 		if (err)
2610 			return (err);
2611 		ifsd_flags[pidx] |= TX_SW_DESC_MAPPED;
2612 		i = 0;
2613 		next = pidx;
2614 		mask = (txq->ift_size-1);
2615 		m = *m0;
2616 		do {
2617 			mp = &ifsd_m[next];
2618 			*mp = m;
2619 			m = m->m_next;
2620 			if (__predict_false((*mp)->m_len == 0)) {
2621 				m_free(*mp);
2622 				*mp = NULL;
2623 			} else
2624 				next = (pidx + i) & (ntxd-1);
2625 		} while (m != NULL);
2626 	} else {
2627 		int buflen, sgsize, max_sgsize;
2628 		vm_offset_t vaddr;
2629 		vm_paddr_t curaddr;
2630 
2631 		count = i = 0;
2632 		maxsegsz = sctx->isc_tx_maxsize;
2633 		m = *m0;
2634 		do {
2635 			if (__predict_false(m->m_len <= 0)) {
2636 				tmp = m;
2637 				m = m->m_next;
2638 				tmp->m_next = NULL;
2639 				m_free(tmp);
2640 				continue;
2641 			}
2642 			buflen = m->m_len;
2643 			vaddr = (vm_offset_t)m->m_data;
2644 			/*
2645 			 * see if we can't be smarter about physically
2646 			 * contiguous mappings
2647 			 */
2648 			next = (pidx + count) & (ntxd-1);
2649 			MPASS(ifsd_m[next] == NULL);
2650 #if MEMORY_LOGGING
2651 			txq->ift_enqueued++;
2652 #endif
2653 			ifsd_m[next] = m;
2654 			while (buflen > 0) {
2655 				max_sgsize = MIN(buflen, maxsegsz);
2656 				curaddr = pmap_kextract(vaddr);
2657 				sgsize = PAGE_SIZE - (curaddr & PAGE_MASK);
2658 				sgsize = MIN(sgsize, max_sgsize);
2659 				segs[i].ds_addr = curaddr;
2660 				segs[i].ds_len = sgsize;
2661 				vaddr += sgsize;
2662 				buflen -= sgsize;
2663 				i++;
2664 				if (i >= max_segs)
2665 					goto err;
2666 			}
2667 			count++;
2668 			tmp = m;
2669 			m = m->m_next;
2670 		} while (m != NULL);
2671 		*nsegs = i;
2672 	}
2673 	return (0);
2674 err:
2675 	*m0 = iflib_remove_mbuf(txq);
2676 	return (EFBIG);
2677 }
2678 
2679 static int
2680 iflib_encap(iflib_txq_t txq, struct mbuf **m_headp)
2681 {
2682 	if_ctx_t		ctx;
2683 	if_shared_ctx_t		sctx;
2684 	if_softc_ctx_t		scctx;
2685 	bus_dma_segment_t	*segs;
2686 	struct mbuf		*m_head;
2687 	bus_dmamap_t		map;
2688 	struct if_pkt_info	pi;
2689 	int remap = 0;
2690 	int err, nsegs, ndesc, max_segs, pidx, cidx, next, ntxd;
2691 	bus_dma_tag_t desc_tag;
2692 
2693 	segs = txq->ift_segs;
2694 	ctx = txq->ift_ctx;
2695 	sctx = ctx->ifc_sctx;
2696 	scctx = &ctx->ifc_softc_ctx;
2697 	segs = txq->ift_segs;
2698 	ntxd = txq->ift_size;
2699 	m_head = *m_headp;
2700 	map = NULL;
2701 
2702 	/*
2703 	 * If we're doing TSO the next descriptor to clean may be quite far ahead
2704 	 */
2705 	cidx = txq->ift_cidx;
2706 	pidx = txq->ift_pidx;
2707 	next = (cidx + CACHE_PTR_INCREMENT) & (ntxd-1);
2708 
2709 	/* prefetch the next cache line of mbuf pointers and flags */
2710 	prefetch(&txq->ift_sds.ifsd_m[next]);
2711 	if (txq->ift_sds.ifsd_map != NULL) {
2712 		prefetch(&txq->ift_sds.ifsd_map[next]);
2713 		map = txq->ift_sds.ifsd_map[pidx];
2714 		next = (cidx + CACHE_LINE_SIZE) & (ntxd-1);
2715 		prefetch(&txq->ift_sds.ifsd_flags[next]);
2716 	}
2717 
2718 
2719 	if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
2720 		desc_tag = txq->ift_tso_desc_tag;
2721 		max_segs = scctx->isc_tx_tso_segments_max;
2722 	} else {
2723 		desc_tag = txq->ift_desc_tag;
2724 		max_segs = scctx->isc_tx_nsegments;
2725 	}
2726 	m_head = *m_headp;
2727 	bzero(&pi, sizeof(pi));
2728 	pi.ipi_len = m_head->m_pkthdr.len;
2729 	pi.ipi_mflags = (m_head->m_flags & (M_VLANTAG|M_BCAST|M_MCAST));
2730 	pi.ipi_csum_flags = m_head->m_pkthdr.csum_flags;
2731 	pi.ipi_vtag = (m_head->m_flags & M_VLANTAG) ? m_head->m_pkthdr.ether_vtag : 0;
2732 	pi.ipi_pidx = pidx;
2733 	pi.ipi_qsidx = txq->ift_id;
2734 
2735 	/* deliberate bitwise OR to make one condition */
2736 	if (__predict_true((pi.ipi_csum_flags | pi.ipi_vtag))) {
2737 		if (__predict_false((err = iflib_parse_header(txq, &pi, m_headp)) != 0))
2738 			return (err);
2739 		m_head = *m_headp;
2740 	}
2741 
2742 retry:
2743 	err = iflib_busdma_load_mbuf_sg(txq, desc_tag, map, m_headp, segs, &nsegs, max_segs, BUS_DMA_NOWAIT);
2744 defrag:
2745 	if (__predict_false(err)) {
2746 		switch (err) {
2747 		case EFBIG:
2748 			/* try collapse once and defrag once */
2749 			if (remap == 0)
2750 				m_head = m_collapse(*m_headp, M_NOWAIT, max_segs);
2751 			if (remap == 1)
2752 				m_head = m_defrag(*m_headp, M_NOWAIT);
2753 			remap++;
2754 			if (__predict_false(m_head == NULL))
2755 				goto defrag_failed;
2756 			txq->ift_mbuf_defrag++;
2757 			*m_headp = m_head;
2758 			goto retry;
2759 			break;
2760 		case ENOMEM:
2761 			txq->ift_no_tx_dma_setup++;
2762 			break;
2763 		default:
2764 			txq->ift_no_tx_dma_setup++;
2765 			m_freem(*m_headp);
2766 			DBG_COUNTER_INC(tx_frees);
2767 			*m_headp = NULL;
2768 			break;
2769 		}
2770 		txq->ift_map_failed++;
2771 		DBG_COUNTER_INC(encap_load_mbuf_fail);
2772 		return (err);
2773 	}
2774 
2775 	/*
2776 	 * XXX assumes a 1 to 1 relationship between segments and
2777 	 *        descriptors - this does not hold true on all drivers, e.g.
2778 	 *        cxgb
2779 	 */
2780 	if (__predict_false(nsegs + 2 > TXQ_AVAIL(txq))) {
2781 		txq->ift_no_desc_avail++;
2782 		if (map != NULL)
2783 			bus_dmamap_unload(desc_tag, map);
2784 		DBG_COUNTER_INC(encap_txq_avail_fail);
2785 		if ((txq->ift_task.gt_task.ta_flags & TASK_ENQUEUED) == 0)
2786 			GROUPTASK_ENQUEUE(&txq->ift_task);
2787 		return (ENOBUFS);
2788 	}
2789 	pi.ipi_segs = segs;
2790 	pi.ipi_nsegs = nsegs;
2791 
2792 	MPASS(pidx >= 0 && pidx < txq->ift_size);
2793 #ifdef PKT_DEBUG
2794 	print_pkt(&pi);
2795 #endif
2796 	if ((err = ctx->isc_txd_encap(ctx->ifc_softc, &pi)) == 0) {
2797 		bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
2798 						BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2799 
2800 		DBG_COUNTER_INC(tx_encap);
2801 		MPASS(pi.ipi_new_pidx >= 0 &&
2802 		    pi.ipi_new_pidx < txq->ift_size);
2803 
2804 		ndesc = pi.ipi_new_pidx - pi.ipi_pidx;
2805 		if (pi.ipi_new_pidx < pi.ipi_pidx) {
2806 			ndesc += txq->ift_size;
2807 			txq->ift_gen = 1;
2808 		}
2809 		/*
2810 		 * drivers can need as many as
2811 		 * two sentinels
2812 		 */
2813 		MPASS(ndesc <= pi.ipi_nsegs + 2);
2814 		MPASS(pi.ipi_new_pidx != pidx);
2815 		MPASS(ndesc > 0);
2816 		txq->ift_in_use += ndesc;
2817 		/*
2818 		 * We update the last software descriptor again here because there may
2819 		 * be a sentinel and/or there may be more mbufs than segments
2820 		 */
2821 		txq->ift_pidx = pi.ipi_new_pidx;
2822 		txq->ift_npending += pi.ipi_ndescs;
2823 	} else if (__predict_false(err == EFBIG && remap < 2)) {
2824 		*m_headp = m_head = iflib_remove_mbuf(txq);
2825 		remap = 1;
2826 		txq->ift_txd_encap_efbig++;
2827 		goto defrag;
2828 	} else
2829 		DBG_COUNTER_INC(encap_txd_encap_fail);
2830 	return (err);
2831 
2832 defrag_failed:
2833 	txq->ift_mbuf_defrag_failed++;
2834 	txq->ift_map_failed++;
2835 	m_freem(*m_headp);
2836 	DBG_COUNTER_INC(tx_frees);
2837 	*m_headp = NULL;
2838 	return (ENOMEM);
2839 }
2840 
2841 /* forward compatibility for cxgb */
2842 #define FIRST_QSET(ctx) 0
2843 
2844 #define NTXQSETS(ctx) ((ctx)->ifc_softc_ctx.isc_ntxqsets)
2845 #define NRXQSETS(ctx) ((ctx)->ifc_softc_ctx.isc_nrxqsets)
2846 #define QIDX(ctx, m) ((((m)->m_pkthdr.flowid & ctx->ifc_softc_ctx.isc_rss_table_mask) % NTXQSETS(ctx)) + FIRST_QSET(ctx))
2847 #define DESC_RECLAIMABLE(q) ((int)((q)->ift_processed - (q)->ift_cleaned - (q)->ift_ctx->ifc_softc_ctx.isc_tx_nsegments))
2848 #define RECLAIM_THRESH(ctx) ((ctx)->ifc_sctx->isc_tx_reclaim_thresh)
2849 #define MAX_TX_DESC(ctx) ((ctx)->ifc_softc_ctx.isc_tx_tso_segments_max)
2850 
2851 
2852 
2853 /* if there are more than TXQ_MIN_OCCUPANCY packets pending we consider deferring
2854  * doorbell writes
2855  *
2856  * ORing with 2 assures that min occupancy is never less than 2 without any conditional logic
2857  */
2858 #define TXQ_MIN_OCCUPANCY(size) ((size >> 6)| 0x2)
2859 
2860 static inline int
2861 iflib_txq_min_occupancy(iflib_txq_t txq)
2862 {
2863 	if_ctx_t ctx;
2864 
2865 	ctx = txq->ift_ctx;
2866 	return (get_inuse(txq->ift_size, txq->ift_cidx, txq->ift_pidx,
2867 	    txq->ift_gen) < TXQ_MIN_OCCUPANCY(txq->ift_size) +
2868 	    MAX_TX_DESC(ctx));
2869 }
2870 
2871 static void
2872 iflib_tx_desc_free(iflib_txq_t txq, int n)
2873 {
2874 	int hasmap;
2875 	uint32_t qsize, cidx, mask, gen;
2876 	struct mbuf *m, **ifsd_m;
2877 	uint8_t *ifsd_flags;
2878 	bus_dmamap_t *ifsd_map;
2879 
2880 	cidx = txq->ift_cidx;
2881 	gen = txq->ift_gen;
2882 	qsize = txq->ift_size;
2883 	mask = qsize-1;
2884 	hasmap = txq->ift_sds.ifsd_map != NULL;
2885 	ifsd_flags = txq->ift_sds.ifsd_flags;
2886 	ifsd_m = txq->ift_sds.ifsd_m;
2887 	ifsd_map = txq->ift_sds.ifsd_map;
2888 
2889 	while (n--) {
2890 		prefetch(ifsd_m[(cidx + 3) & mask]);
2891 		prefetch(ifsd_m[(cidx + 4) & mask]);
2892 
2893 		if (ifsd_m[cidx] != NULL) {
2894 			prefetch(&ifsd_m[(cidx + CACHE_PTR_INCREMENT) & mask]);
2895 			prefetch(&ifsd_flags[(cidx + CACHE_PTR_INCREMENT) & mask]);
2896 			if (hasmap && (ifsd_flags[cidx] & TX_SW_DESC_MAPPED)) {
2897 				/*
2898 				 * does it matter if it's not the TSO tag? If so we'll
2899 				 * have to add the type to flags
2900 				 */
2901 				bus_dmamap_unload(txq->ift_desc_tag, ifsd_map[cidx]);
2902 				ifsd_flags[cidx] &= ~TX_SW_DESC_MAPPED;
2903 			}
2904 			if ((m = ifsd_m[cidx]) != NULL) {
2905 				/* XXX we don't support any drivers that batch packets yet */
2906 				MPASS(m->m_nextpkt == NULL);
2907 
2908 				m_free(m);
2909 				ifsd_m[cidx] = NULL;
2910 #if MEMORY_LOGGING
2911 				txq->ift_dequeued++;
2912 #endif
2913 				DBG_COUNTER_INC(tx_frees);
2914 			}
2915 		}
2916 		if (__predict_false(++cidx == qsize)) {
2917 			cidx = 0;
2918 			gen = 0;
2919 		}
2920 	}
2921 	txq->ift_cidx = cidx;
2922 	txq->ift_gen = gen;
2923 }
2924 
2925 static __inline int
2926 iflib_completed_tx_reclaim(iflib_txq_t txq, int thresh)
2927 {
2928 	int reclaim;
2929 	if_ctx_t ctx = txq->ift_ctx;
2930 
2931 	KASSERT(thresh >= 0, ("invalid threshold to reclaim"));
2932 	MPASS(thresh /*+ MAX_TX_DESC(txq->ift_ctx) */ < txq->ift_size);
2933 
2934 	/*
2935 	 * Need a rate-limiting check so that this isn't called every time
2936 	 */
2937 	iflib_tx_credits_update(ctx, txq);
2938 	reclaim = DESC_RECLAIMABLE(txq);
2939 
2940 	if (reclaim <= thresh /* + MAX_TX_DESC(txq->ift_ctx) */) {
2941 #ifdef INVARIANTS
2942 		if (iflib_verbose_debug) {
2943 			printf("%s processed=%ju cleaned=%ju tx_nsegments=%d reclaim=%d thresh=%d\n", __FUNCTION__,
2944 			       txq->ift_processed, txq->ift_cleaned, txq->ift_ctx->ifc_softc_ctx.isc_tx_nsegments,
2945 			       reclaim, thresh);
2946 
2947 		}
2948 #endif
2949 		return (0);
2950 	}
2951 	iflib_tx_desc_free(txq, reclaim);
2952 	txq->ift_cleaned += reclaim;
2953 	txq->ift_in_use -= reclaim;
2954 
2955 	if (txq->ift_active == FALSE)
2956 		txq->ift_active = TRUE;
2957 
2958 	return (reclaim);
2959 }
2960 
2961 static struct mbuf **
2962 _ring_peek_one(struct ifmp_ring *r, int cidx, int offset)
2963 {
2964 
2965 	return (__DEVOLATILE(struct mbuf **, &r->items[(cidx + offset) & (r->size-1)]));
2966 }
2967 
2968 static void
2969 iflib_txq_check_drain(iflib_txq_t txq, int budget)
2970 {
2971 
2972 	ifmp_ring_check_drainage(txq->ift_br[0], budget);
2973 }
2974 
2975 static uint32_t
2976 iflib_txq_can_drain(struct ifmp_ring *r)
2977 {
2978 	iflib_txq_t txq = r->cookie;
2979 	if_ctx_t ctx = txq->ift_ctx;
2980 
2981 	return ((TXQ_AVAIL(txq) > MAX_TX_DESC(ctx) + 2) ||
2982 		ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, txq->ift_cidx_processed, false));
2983 }
2984 
2985 static uint32_t
2986 iflib_txq_drain(struct ifmp_ring *r, uint32_t cidx, uint32_t pidx)
2987 {
2988 	iflib_txq_t txq = r->cookie;
2989 	if_ctx_t ctx = txq->ift_ctx;
2990 	if_t ifp = ctx->ifc_ifp;
2991 	struct mbuf **mp, *m;
2992 	int i, count, consumed, pkt_sent, bytes_sent, mcast_sent, avail, err, in_use_prev, desc_used;
2993 
2994 	if (__predict_false(!(if_getdrvflags(ifp) & IFF_DRV_RUNNING) ||
2995 			    !LINK_ACTIVE(ctx))) {
2996 		DBG_COUNTER_INC(txq_drain_notready);
2997 		return (0);
2998 	}
2999 
3000 	avail = IDXDIFF(pidx, cidx, r->size);
3001 	if (__predict_false(ctx->ifc_flags & IFC_QFLUSH)) {
3002 		DBG_COUNTER_INC(txq_drain_flushing);
3003 		for (i = 0; i < avail; i++) {
3004 			m_free(r->items[(cidx + i) & (r->size-1)]);
3005 			r->items[(cidx + i) & (r->size-1)] = NULL;
3006 		}
3007 		return (avail);
3008 	}
3009 	iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx));
3010 	if (__predict_false(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE)) {
3011 		txq->ift_qstatus = IFLIB_QUEUE_IDLE;
3012 		CALLOUT_LOCK(txq);
3013 		callout_stop(&txq->ift_timer);
3014 		callout_stop(&txq->ift_db_check);
3015 		CALLOUT_UNLOCK(txq);
3016 		DBG_COUNTER_INC(txq_drain_oactive);
3017 		return (0);
3018 	}
3019 	consumed = mcast_sent = bytes_sent = pkt_sent = 0;
3020 	count = MIN(avail, TX_BATCH_SIZE);
3021 #ifdef INVARIANTS
3022 	if (iflib_verbose_debug)
3023 		printf("%s avail=%d ifc_flags=%x txq_avail=%d ", __FUNCTION__,
3024 		       avail, ctx->ifc_flags, TXQ_AVAIL(txq));
3025 #endif
3026 
3027 	for (desc_used = i = 0; i < count && TXQ_AVAIL(txq) > MAX_TX_DESC(ctx) + 2; i++) {
3028 		mp = _ring_peek_one(r, cidx, i);
3029 		MPASS(mp != NULL && *mp != NULL);
3030 		in_use_prev = txq->ift_in_use;
3031 		if ((err = iflib_encap(txq, mp)) == ENOBUFS) {
3032 			DBG_COUNTER_INC(txq_drain_encapfail);
3033 			/* no room - bail out */
3034 			break;
3035 		}
3036 		consumed++;
3037 		if (err) {
3038 			DBG_COUNTER_INC(txq_drain_encapfail);
3039 			/* we can't send this packet - skip it */
3040 			continue;
3041 		}
3042 		pkt_sent++;
3043 		m = *mp;
3044 		DBG_COUNTER_INC(tx_sent);
3045 		bytes_sent += m->m_pkthdr.len;
3046 		if (m->m_flags & M_MCAST)
3047 			mcast_sent++;
3048 
3049 		txq->ift_db_pending += (txq->ift_in_use - in_use_prev);
3050 		desc_used += (txq->ift_in_use - in_use_prev);
3051 		iflib_txd_db_check(ctx, txq, FALSE);
3052 		ETHER_BPF_MTAP(ifp, m);
3053 		if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)))
3054 			break;
3055 
3056 		if (desc_used >= TXQ_MAX_DB_CONSUMED(txq->ift_size))
3057 			break;
3058 	}
3059 
3060 	if ((iflib_min_tx_latency || iflib_txq_min_occupancy(txq)) && txq->ift_db_pending)
3061 		iflib_txd_db_check(ctx, txq, TRUE);
3062 	else if ((txq->ift_db_pending || TXQ_AVAIL(txq) <= MAX_TX_DESC(ctx) + 2) &&
3063 		 (callout_pending(&txq->ift_db_check) == 0)) {
3064 		txq->ift_db_pending_queued = txq->ift_db_pending;
3065 		callout_reset_on(&txq->ift_db_check, 1, iflib_txd_deferred_db_check,
3066 				 txq, txq->ift_db_check.c_cpu);
3067 	}
3068 	if_inc_counter(ifp, IFCOUNTER_OBYTES, bytes_sent);
3069 	if_inc_counter(ifp, IFCOUNTER_OPACKETS, pkt_sent);
3070 	if (mcast_sent)
3071 		if_inc_counter(ifp, IFCOUNTER_OMCASTS, mcast_sent);
3072 #ifdef INVARIANTS
3073 	if (iflib_verbose_debug)
3074 		printf("consumed=%d\n", consumed);
3075 #endif
3076 	return (consumed);
3077 }
3078 
3079 static uint32_t
3080 iflib_txq_drain_always(struct ifmp_ring *r)
3081 {
3082 	return (1);
3083 }
3084 
3085 static uint32_t
3086 iflib_txq_drain_free(struct ifmp_ring *r, uint32_t cidx, uint32_t pidx)
3087 {
3088 	int i, avail;
3089 	struct mbuf **mp;
3090 	iflib_txq_t txq;
3091 
3092 	txq = r->cookie;
3093 
3094 	txq->ift_qstatus = IFLIB_QUEUE_IDLE;
3095 	CALLOUT_LOCK(txq);
3096 	callout_stop(&txq->ift_timer);
3097 	callout_stop(&txq->ift_db_check);
3098 	CALLOUT_UNLOCK(txq);
3099 
3100 	avail = IDXDIFF(pidx, cidx, r->size);
3101 	for (i = 0; i < avail; i++) {
3102 		mp = _ring_peek_one(r, cidx, i);
3103 		m_freem(*mp);
3104 	}
3105 	MPASS(ifmp_ring_is_stalled(r) == 0);
3106 	return (avail);
3107 }
3108 
3109 static void
3110 iflib_ifmp_purge(iflib_txq_t txq)
3111 {
3112 	struct ifmp_ring *r;
3113 
3114 	r = txq->ift_br[0];
3115 	r->drain = iflib_txq_drain_free;
3116 	r->can_drain = iflib_txq_drain_always;
3117 
3118 	ifmp_ring_check_drainage(r, r->size);
3119 
3120 	r->drain = iflib_txq_drain;
3121 	r->can_drain = iflib_txq_can_drain;
3122 }
3123 
3124 static void
3125 _task_fn_tx(void *context)
3126 {
3127 	iflib_txq_t txq = context;
3128 	if_ctx_t ctx = txq->ift_ctx;
3129 
3130 #ifdef IFLIB_DIAGNOSTICS
3131 	txq->ift_cpu_exec_count[curcpu]++;
3132 #endif
3133 	if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
3134 		return;
3135 	ifmp_ring_check_drainage(txq->ift_br[0], TX_BATCH_SIZE);
3136 }
3137 
3138 static void
3139 _task_fn_rx(void *context)
3140 {
3141 	iflib_rxq_t rxq = context;
3142 	if_ctx_t ctx = rxq->ifr_ctx;
3143 	bool more;
3144 	int rc;
3145 
3146 #ifdef IFLIB_DIAGNOSTICS
3147 	rxq->ifr_cpu_exec_count[curcpu]++;
3148 #endif
3149 	DBG_COUNTER_INC(task_fn_rxs);
3150 	if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)))
3151 		return;
3152 
3153 	if ((more = iflib_rxeof(rxq, 16 /* XXX */)) == false) {
3154 		if (ctx->ifc_flags & IFC_LEGACY)
3155 			IFDI_INTR_ENABLE(ctx);
3156 		else {
3157 			DBG_COUNTER_INC(rx_intr_enables);
3158 			rc = IFDI_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id);
3159 			KASSERT(rc != ENOTSUP, ("MSI-X support requires queue_intr_enable, but not implemented in driver"));
3160 		}
3161 	}
3162 	if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)))
3163 		return;
3164 	if (more)
3165 		GROUPTASK_ENQUEUE(&rxq->ifr_task);
3166 }
3167 
3168 static void
3169 _task_fn_admin(void *context)
3170 {
3171 	if_ctx_t ctx = context;
3172 	if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
3173 	iflib_txq_t txq;
3174 	int i;
3175 
3176 	if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
3177 		return;
3178 
3179 	CTX_LOCK(ctx);
3180 	for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) {
3181 		CALLOUT_LOCK(txq);
3182 		callout_stop(&txq->ift_timer);
3183 		CALLOUT_UNLOCK(txq);
3184 	}
3185 	IFDI_UPDATE_ADMIN_STATUS(ctx);
3186 	for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++)
3187 		callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, txq->ift_timer.c_cpu);
3188 	IFDI_LINK_INTR_ENABLE(ctx);
3189 	CTX_UNLOCK(ctx);
3190 
3191 	if (LINK_ACTIVE(ctx) == 0)
3192 		return;
3193 	for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++)
3194 		iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET);
3195 }
3196 
3197 
3198 static void
3199 _task_fn_iov(void *context)
3200 {
3201 	if_ctx_t ctx = context;
3202 
3203 	if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
3204 		return;
3205 
3206 	CTX_LOCK(ctx);
3207 	IFDI_VFLR_HANDLE(ctx);
3208 	CTX_UNLOCK(ctx);
3209 }
3210 
3211 static int
3212 iflib_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
3213 {
3214 	int err;
3215 	if_int_delay_info_t info;
3216 	if_ctx_t ctx;
3217 
3218 	info = (if_int_delay_info_t)arg1;
3219 	ctx = info->iidi_ctx;
3220 	info->iidi_req = req;
3221 	info->iidi_oidp = oidp;
3222 	CTX_LOCK(ctx);
3223 	err = IFDI_SYSCTL_INT_DELAY(ctx, info);
3224 	CTX_UNLOCK(ctx);
3225 	return (err);
3226 }
3227 
3228 /*********************************************************************
3229  *
3230  *  IFNET FUNCTIONS
3231  *
3232  **********************************************************************/
3233 
3234 static void
3235 iflib_if_init_locked(if_ctx_t ctx)
3236 {
3237 	iflib_stop(ctx);
3238 	iflib_init_locked(ctx);
3239 }
3240 
3241 
3242 static void
3243 iflib_if_init(void *arg)
3244 {
3245 	if_ctx_t ctx = arg;
3246 
3247 	CTX_LOCK(ctx);
3248 	iflib_if_init_locked(ctx);
3249 	CTX_UNLOCK(ctx);
3250 }
3251 
3252 static int
3253 iflib_if_transmit(if_t ifp, struct mbuf *m)
3254 {
3255 	if_ctx_t	ctx = if_getsoftc(ifp);
3256 
3257 	iflib_txq_t txq;
3258 	int err, qidx;
3259 
3260 	if (__predict_false((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || !LINK_ACTIVE(ctx))) {
3261 		DBG_COUNTER_INC(tx_frees);
3262 		m_freem(m);
3263 		return (ENOBUFS);
3264 	}
3265 
3266 	MPASS(m->m_nextpkt == NULL);
3267 	qidx = 0;
3268 	if ((NTXQSETS(ctx) > 1) && M_HASHTYPE_GET(m))
3269 		qidx = QIDX(ctx, m);
3270 	/*
3271 	 * XXX calculate buf_ring based on flowid (divvy up bits?)
3272 	 */
3273 	txq = &ctx->ifc_txqs[qidx];
3274 
3275 #ifdef DRIVER_BACKPRESSURE
3276 	if (txq->ift_closed) {
3277 		while (m != NULL) {
3278 			next = m->m_nextpkt;
3279 			m->m_nextpkt = NULL;
3280 			m_freem(m);
3281 			m = next;
3282 		}
3283 		return (ENOBUFS);
3284 	}
3285 #endif
3286 #ifdef notyet
3287 	qidx = count = 0;
3288 	mp = marr;
3289 	next = m;
3290 	do {
3291 		count++;
3292 		next = next->m_nextpkt;
3293 	} while (next != NULL);
3294 
3295 	if (count > nitems(marr))
3296 		if ((mp = malloc(count*sizeof(struct mbuf *), M_IFLIB, M_NOWAIT)) == NULL) {
3297 			/* XXX check nextpkt */
3298 			m_freem(m);
3299 			/* XXX simplify for now */
3300 			DBG_COUNTER_INC(tx_frees);
3301 			return (ENOBUFS);
3302 		}
3303 	for (next = m, i = 0; next != NULL; i++) {
3304 		mp[i] = next;
3305 		next = next->m_nextpkt;
3306 		mp[i]->m_nextpkt = NULL;
3307 	}
3308 #endif
3309 	DBG_COUNTER_INC(tx_seen);
3310 	err = ifmp_ring_enqueue(txq->ift_br[0], (void **)&m, 1, TX_BATCH_SIZE);
3311 
3312 	if (err) {
3313 		GROUPTASK_ENQUEUE(&txq->ift_task);
3314 		/* support forthcoming later */
3315 #ifdef DRIVER_BACKPRESSURE
3316 		txq->ift_closed = TRUE;
3317 #endif
3318 		ifmp_ring_check_drainage(txq->ift_br[0], TX_BATCH_SIZE);
3319 		m_freem(m);
3320 	} else if (TXQ_AVAIL(txq) < (txq->ift_size >> 1)) {
3321 		GROUPTASK_ENQUEUE(&txq->ift_task);
3322 	}
3323 
3324 	return (err);
3325 }
3326 
3327 static void
3328 iflib_if_qflush(if_t ifp)
3329 {
3330 	if_ctx_t ctx = if_getsoftc(ifp);
3331 	iflib_txq_t txq = ctx->ifc_txqs;
3332 	int i;
3333 
3334 	CTX_LOCK(ctx);
3335 	ctx->ifc_flags |= IFC_QFLUSH;
3336 	CTX_UNLOCK(ctx);
3337 	for (i = 0; i < NTXQSETS(ctx); i++, txq++)
3338 		while (!(ifmp_ring_is_idle(txq->ift_br[0]) || ifmp_ring_is_stalled(txq->ift_br[0])))
3339 			iflib_txq_check_drain(txq, 0);
3340 	CTX_LOCK(ctx);
3341 	ctx->ifc_flags &= ~IFC_QFLUSH;
3342 	CTX_UNLOCK(ctx);
3343 
3344 	if_qflush(ifp);
3345 }
3346 
3347 
3348 #define IFCAP_FLAGS (IFCAP_TXCSUM_IPV6 | IFCAP_RXCSUM_IPV6 | IFCAP_HWCSUM | IFCAP_LRO | \
3349 		     IFCAP_TSO4 | IFCAP_TSO6 | IFCAP_VLAN_HWTAGGING |	\
3350 		     IFCAP_VLAN_MTU | IFCAP_VLAN_HWFILTER | IFCAP_VLAN_HWTSO)
3351 
3352 static int
3353 iflib_if_ioctl(if_t ifp, u_long command, caddr_t data)
3354 {
3355 	if_ctx_t ctx = if_getsoftc(ifp);
3356 	struct ifreq	*ifr = (struct ifreq *)data;
3357 #if defined(INET) || defined(INET6)
3358 	struct ifaddr	*ifa = (struct ifaddr *)data;
3359 #endif
3360 	bool		avoid_reset = FALSE;
3361 	int		err = 0, reinit = 0, bits;
3362 
3363 	switch (command) {
3364 	case SIOCSIFADDR:
3365 #ifdef INET
3366 		if (ifa->ifa_addr->sa_family == AF_INET)
3367 			avoid_reset = TRUE;
3368 #endif
3369 #ifdef INET6
3370 		if (ifa->ifa_addr->sa_family == AF_INET6)
3371 			avoid_reset = TRUE;
3372 #endif
3373 		/*
3374 		** Calling init results in link renegotiation,
3375 		** so we avoid doing it when possible.
3376 		*/
3377 		if (avoid_reset) {
3378 			if_setflagbits(ifp, IFF_UP,0);
3379 			if (!(if_getdrvflags(ifp)& IFF_DRV_RUNNING))
3380 				reinit = 1;
3381 #ifdef INET
3382 			if (!(if_getflags(ifp) & IFF_NOARP))
3383 				arp_ifinit(ifp, ifa);
3384 #endif
3385 		} else
3386 			err = ether_ioctl(ifp, command, data);
3387 		break;
3388 	case SIOCSIFMTU:
3389 		CTX_LOCK(ctx);
3390 		if (ifr->ifr_mtu == if_getmtu(ifp)) {
3391 			CTX_UNLOCK(ctx);
3392 			break;
3393 		}
3394 		bits = if_getdrvflags(ifp);
3395 		/* stop the driver and free any clusters before proceeding */
3396 		iflib_stop(ctx);
3397 
3398 		if ((err = IFDI_MTU_SET(ctx, ifr->ifr_mtu)) == 0) {
3399 			if (ifr->ifr_mtu > ctx->ifc_max_fl_buf_size)
3400 				ctx->ifc_flags |= IFC_MULTISEG;
3401 			else
3402 				ctx->ifc_flags &= ~IFC_MULTISEG;
3403 			err = if_setmtu(ifp, ifr->ifr_mtu);
3404 		}
3405 		iflib_init_locked(ctx);
3406 		if_setdrvflags(ifp, bits);
3407 		CTX_UNLOCK(ctx);
3408 		break;
3409 	case SIOCSIFFLAGS:
3410 		CTX_LOCK(ctx);
3411 		if (if_getflags(ifp) & IFF_UP) {
3412 			if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
3413 				if ((if_getflags(ifp) ^ ctx->ifc_if_flags) &
3414 				    (IFF_PROMISC | IFF_ALLMULTI)) {
3415 					err = IFDI_PROMISC_SET(ctx, if_getflags(ifp));
3416 				}
3417 			} else
3418 				reinit = 1;
3419 		} else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
3420 			iflib_stop(ctx);
3421 		}
3422 		ctx->ifc_if_flags = if_getflags(ifp);
3423 		CTX_UNLOCK(ctx);
3424 		break;
3425 	case SIOCADDMULTI:
3426 	case SIOCDELMULTI:
3427 		if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
3428 			CTX_LOCK(ctx);
3429 			IFDI_INTR_DISABLE(ctx);
3430 			IFDI_MULTI_SET(ctx);
3431 			IFDI_INTR_ENABLE(ctx);
3432 			CTX_UNLOCK(ctx);
3433 		}
3434 		break;
3435 	case SIOCSIFMEDIA:
3436 		CTX_LOCK(ctx);
3437 		IFDI_MEDIA_SET(ctx);
3438 		CTX_UNLOCK(ctx);
3439 		/* falls thru */
3440 	case SIOCGIFMEDIA:
3441 		err = ifmedia_ioctl(ifp, ifr, &ctx->ifc_media, command);
3442 		break;
3443 	case SIOCGI2C:
3444 	{
3445 		struct ifi2creq i2c;
3446 
3447 		err = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
3448 		if (err != 0)
3449 			break;
3450 		if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
3451 			err = EINVAL;
3452 			break;
3453 		}
3454 		if (i2c.len > sizeof(i2c.data)) {
3455 			err = EINVAL;
3456 			break;
3457 		}
3458 
3459 		if ((err = IFDI_I2C_REQ(ctx, &i2c)) == 0)
3460 			err = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
3461 		break;
3462 	}
3463 	case SIOCSIFCAP:
3464 	{
3465 		int mask, setmask;
3466 
3467 		mask = ifr->ifr_reqcap ^ if_getcapenable(ifp);
3468 		setmask = 0;
3469 #ifdef TCP_OFFLOAD
3470 		setmask |= mask & (IFCAP_TOE4|IFCAP_TOE6);
3471 #endif
3472 		setmask |= (mask & IFCAP_FLAGS);
3473 
3474 		if (setmask  & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6))
3475 			setmask |= (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6);
3476 		if ((mask & IFCAP_WOL) &&
3477 		    (if_getcapabilities(ifp) & IFCAP_WOL) != 0)
3478 			setmask |= (mask & (IFCAP_WOL_MCAST|IFCAP_WOL_MAGIC));
3479 		if_vlancap(ifp);
3480 		/*
3481 		 * want to ensure that traffic has stopped before we change any of the flags
3482 		 */
3483 		if (setmask) {
3484 			CTX_LOCK(ctx);
3485 			bits = if_getdrvflags(ifp);
3486 			if (bits & IFF_DRV_RUNNING)
3487 				iflib_stop(ctx);
3488 			if_togglecapenable(ifp, setmask);
3489 			if (bits & IFF_DRV_RUNNING)
3490 				iflib_init_locked(ctx);
3491 			if_setdrvflags(ifp, bits);
3492 			CTX_UNLOCK(ctx);
3493 		}
3494 		break;
3495 	    }
3496 	case SIOCGPRIVATE_0:
3497 	case SIOCSDRVSPEC:
3498 	case SIOCGDRVSPEC:
3499 		CTX_LOCK(ctx);
3500 		err = IFDI_PRIV_IOCTL(ctx, command, data);
3501 		CTX_UNLOCK(ctx);
3502 		break;
3503 	default:
3504 		err = ether_ioctl(ifp, command, data);
3505 		break;
3506 	}
3507 	if (reinit)
3508 		iflib_if_init(ctx);
3509 	return (err);
3510 }
3511 
3512 static uint64_t
3513 iflib_if_get_counter(if_t ifp, ift_counter cnt)
3514 {
3515 	if_ctx_t ctx = if_getsoftc(ifp);
3516 
3517 	return (IFDI_GET_COUNTER(ctx, cnt));
3518 }
3519 
3520 /*********************************************************************
3521  *
3522  *  OTHER FUNCTIONS EXPORTED TO THE STACK
3523  *
3524  **********************************************************************/
3525 
3526 static void
3527 iflib_vlan_register(void *arg, if_t ifp, uint16_t vtag)
3528 {
3529 	if_ctx_t ctx = if_getsoftc(ifp);
3530 
3531 	if ((void *)ctx != arg)
3532 		return;
3533 
3534 	if ((vtag == 0) || (vtag > 4095))
3535 		return;
3536 
3537 	CTX_LOCK(ctx);
3538 	IFDI_VLAN_REGISTER(ctx, vtag);
3539 	/* Re-init to load the changes */
3540 	if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
3541 		iflib_init_locked(ctx);
3542 	CTX_UNLOCK(ctx);
3543 }
3544 
3545 static void
3546 iflib_vlan_unregister(void *arg, if_t ifp, uint16_t vtag)
3547 {
3548 	if_ctx_t ctx = if_getsoftc(ifp);
3549 
3550 	if ((void *)ctx != arg)
3551 		return;
3552 
3553 	if ((vtag == 0) || (vtag > 4095))
3554 		return;
3555 
3556 	CTX_LOCK(ctx);
3557 	IFDI_VLAN_UNREGISTER(ctx, vtag);
3558 	/* Re-init to load the changes */
3559 	if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
3560 		iflib_init_locked(ctx);
3561 	CTX_UNLOCK(ctx);
3562 }
3563 
3564 static void
3565 iflib_led_func(void *arg, int onoff)
3566 {
3567 	if_ctx_t ctx = arg;
3568 
3569 	CTX_LOCK(ctx);
3570 	IFDI_LED_FUNC(ctx, onoff);
3571 	CTX_UNLOCK(ctx);
3572 }
3573 
3574 /*********************************************************************
3575  *
3576  *  BUS FUNCTION DEFINITIONS
3577  *
3578  **********************************************************************/
3579 
3580 int
3581 iflib_device_probe(device_t dev)
3582 {
3583 	pci_vendor_info_t *ent;
3584 
3585 	uint16_t	pci_vendor_id, pci_device_id;
3586 	uint16_t	pci_subvendor_id, pci_subdevice_id;
3587 	uint16_t	pci_rev_id;
3588 	if_shared_ctx_t sctx;
3589 
3590 	if ((sctx = DEVICE_REGISTER(dev)) == NULL || sctx->isc_magic != IFLIB_MAGIC)
3591 		return (ENOTSUP);
3592 
3593 	pci_vendor_id = pci_get_vendor(dev);
3594 	pci_device_id = pci_get_device(dev);
3595 	pci_subvendor_id = pci_get_subvendor(dev);
3596 	pci_subdevice_id = pci_get_subdevice(dev);
3597 	pci_rev_id = pci_get_revid(dev);
3598 	if (sctx->isc_parse_devinfo != NULL)
3599 		sctx->isc_parse_devinfo(&pci_device_id, &pci_subvendor_id, &pci_subdevice_id, &pci_rev_id);
3600 
3601 	ent = sctx->isc_vendor_info;
3602 	while (ent->pvi_vendor_id != 0) {
3603 		if (pci_vendor_id != ent->pvi_vendor_id) {
3604 			ent++;
3605 			continue;
3606 		}
3607 		if ((pci_device_id == ent->pvi_device_id) &&
3608 		    ((pci_subvendor_id == ent->pvi_subvendor_id) ||
3609 		     (ent->pvi_subvendor_id == 0)) &&
3610 		    ((pci_subdevice_id == ent->pvi_subdevice_id) ||
3611 		     (ent->pvi_subdevice_id == 0)) &&
3612 		    ((pci_rev_id == ent->pvi_rev_id) ||
3613 		     (ent->pvi_rev_id == 0))) {
3614 
3615 			device_set_desc_copy(dev, ent->pvi_name);
3616 			/* this needs to be changed to zero if the bus probing code
3617 			 * ever stops re-probing on best match because the sctx
3618 			 * may have its values over written by register calls
3619 			 * in subsequent probes
3620 			 */
3621 			return (BUS_PROBE_DEFAULT);
3622 		}
3623 		ent++;
3624 	}
3625 	return (ENXIO);
3626 }
3627 
3628 int
3629 iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ctxp)
3630 {
3631 	int err, rid, msix, msix_bar;
3632 	if_ctx_t ctx;
3633 	if_t ifp;
3634 	if_softc_ctx_t scctx;
3635 	int i;
3636 	uint16_t main_txq;
3637 	uint16_t main_rxq;
3638 
3639 
3640 	ctx = malloc(sizeof(* ctx), M_IFLIB, M_WAITOK|M_ZERO);
3641 
3642 	if (sc == NULL) {
3643 		sc = malloc(sctx->isc_driver->size, M_IFLIB, M_WAITOK|M_ZERO);
3644 		device_set_softc(dev, ctx);
3645 		ctx->ifc_flags |= IFC_SC_ALLOCATED;
3646 	}
3647 
3648 	ctx->ifc_sctx = sctx;
3649 	ctx->ifc_dev = dev;
3650 	ctx->ifc_softc = sc;
3651 
3652 	if ((err = iflib_register(ctx)) != 0) {
3653 		device_printf(dev, "iflib_register failed %d\n", err);
3654 		return (err);
3655 	}
3656 	iflib_add_device_sysctl_pre(ctx);
3657 
3658 	scctx = &ctx->ifc_softc_ctx;
3659 	ifp = ctx->ifc_ifp;
3660 
3661 	/*
3662 	 * XXX sanity check that ntxd & nrxd are a power of 2
3663 	 */
3664 	if (ctx->ifc_sysctl_ntxqs != 0)
3665 		scctx->isc_ntxqsets = ctx->ifc_sysctl_ntxqs;
3666 	if (ctx->ifc_sysctl_nrxqs != 0)
3667 		scctx->isc_nrxqsets = ctx->ifc_sysctl_nrxqs;
3668 
3669 	for (i = 0; i < sctx->isc_ntxqs; i++) {
3670 		if (ctx->ifc_sysctl_ntxds[i] != 0)
3671 			scctx->isc_ntxd[i] = ctx->ifc_sysctl_ntxds[i];
3672 		else
3673 			scctx->isc_ntxd[i] = sctx->isc_ntxd_default[i];
3674 	}
3675 
3676 	for (i = 0; i < sctx->isc_nrxqs; i++) {
3677 		if (ctx->ifc_sysctl_nrxds[i] != 0)
3678 			scctx->isc_nrxd[i] = ctx->ifc_sysctl_nrxds[i];
3679 		else
3680 			scctx->isc_nrxd[i] = sctx->isc_nrxd_default[i];
3681 	}
3682 
3683 	for (i = 0; i < sctx->isc_nrxqs; i++) {
3684 		if (scctx->isc_nrxd[i] < sctx->isc_nrxd_min[i]) {
3685 			device_printf(dev, "nrxd%d: %d less than nrxd_min %d - resetting to min\n",
3686 				      i, scctx->isc_nrxd[i], sctx->isc_nrxd_min[i]);
3687 			scctx->isc_nrxd[i] = sctx->isc_nrxd_min[i];
3688 		}
3689 		if (scctx->isc_nrxd[i] > sctx->isc_nrxd_max[i]) {
3690 			device_printf(dev, "nrxd%d: %d greater than nrxd_max %d - resetting to max\n",
3691 				      i, scctx->isc_nrxd[i], sctx->isc_nrxd_max[i]);
3692 			scctx->isc_nrxd[i] = sctx->isc_nrxd_max[i];
3693 		}
3694 	}
3695 
3696 	for (i = 0; i < sctx->isc_ntxqs; i++) {
3697 		if (scctx->isc_ntxd[i] < sctx->isc_ntxd_min[i]) {
3698 			device_printf(dev, "ntxd%d: %d less than ntxd_min %d - resetting to min\n",
3699 				      i, scctx->isc_ntxd[i], sctx->isc_ntxd_min[i]);
3700 			scctx->isc_ntxd[i] = sctx->isc_ntxd_min[i];
3701 		}
3702 		if (scctx->isc_ntxd[i] > sctx->isc_ntxd_max[i]) {
3703 			device_printf(dev, "ntxd%d: %d greater than ntxd_max %d - resetting to max\n",
3704 				      i, scctx->isc_ntxd[i], sctx->isc_ntxd_max[i]);
3705 			scctx->isc_ntxd[i] = sctx->isc_ntxd_max[i];
3706 		}
3707 	}
3708 
3709 	if ((err = IFDI_ATTACH_PRE(ctx)) != 0) {
3710 		device_printf(dev, "IFDI_ATTACH_PRE failed %d\n", err);
3711 		return (err);
3712 	}
3713 	_iflib_pre_assert(scctx);
3714 	ctx->ifc_txrx = *scctx->isc_txrx;
3715 
3716 #ifdef INVARIANTS
3717 	MPASS(scctx->isc_capenable);
3718 	if (scctx->isc_capenable & IFCAP_TXCSUM)
3719 		MPASS(scctx->isc_tx_csum_flags);
3720 #endif
3721 
3722 	if_setcapabilities(ifp, scctx->isc_capenable);
3723 	if_setcapenable(ifp, scctx->isc_capenable);
3724 
3725 	if (scctx->isc_ntxqsets == 0 || (scctx->isc_ntxqsets_max && scctx->isc_ntxqsets_max < scctx->isc_ntxqsets))
3726 		scctx->isc_ntxqsets = scctx->isc_ntxqsets_max;
3727 	if (scctx->isc_nrxqsets == 0 || (scctx->isc_nrxqsets_max && scctx->isc_nrxqsets_max < scctx->isc_nrxqsets))
3728 		scctx->isc_nrxqsets = scctx->isc_nrxqsets_max;
3729 
3730 #ifdef ACPI_DMAR
3731 	if (dmar_get_dma_tag(device_get_parent(dev), dev) != NULL)
3732 		ctx->ifc_flags |= IFC_DMAR;
3733 #endif
3734 
3735 	msix_bar = scctx->isc_msix_bar;
3736 
3737 	if(sctx->isc_flags & IFLIB_HAS_TXCQ)
3738 		main_txq = 1;
3739 	else
3740 		main_txq = 0;
3741 
3742 	if(sctx->isc_flags & IFLIB_HAS_RXCQ)
3743 		main_rxq = 1;
3744 	else
3745 		main_rxq = 0;
3746 
3747 	/* XXX change for per-queue sizes */
3748 	device_printf(dev, "using %d tx descriptors and %d rx descriptors\n",
3749 		      scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]);
3750 	for (i = 0; i < sctx->isc_nrxqs; i++) {
3751 		if (!powerof2(scctx->isc_nrxd[i])) {
3752 			/* round down instead? */
3753 			device_printf(dev, "# rx descriptors must be a power of 2\n");
3754 			err = EINVAL;
3755 			goto fail;
3756 		}
3757 	}
3758 	for (i = 0; i < sctx->isc_ntxqs; i++) {
3759 		if (!powerof2(scctx->isc_ntxd[i])) {
3760 			device_printf(dev,
3761 			    "# tx descriptors must be a power of 2");
3762 			err = EINVAL;
3763 			goto fail;
3764 		}
3765 	}
3766 
3767 	if (scctx->isc_tx_nsegments > scctx->isc_ntxd[main_txq] /
3768 	    MAX_SINGLE_PACKET_FRACTION)
3769 		scctx->isc_tx_nsegments = max(1, scctx->isc_ntxd[main_txq] /
3770 		    MAX_SINGLE_PACKET_FRACTION);
3771 	if (scctx->isc_tx_tso_segments_max > scctx->isc_ntxd[main_txq] /
3772 	    MAX_SINGLE_PACKET_FRACTION)
3773 		scctx->isc_tx_tso_segments_max = max(1,
3774 		    scctx->isc_ntxd[main_txq] / MAX_SINGLE_PACKET_FRACTION);
3775 
3776 	/*
3777 	 * Protect the stack against modern hardware
3778 	 */
3779 	if (scctx->isc_tx_tso_size_max > FREEBSD_TSO_SIZE_MAX)
3780 		scctx->isc_tx_tso_size_max = FREEBSD_TSO_SIZE_MAX;
3781 
3782 	/* TSO parameters - dig these out of the data sheet - simply correspond to tag setup */
3783 	ifp->if_hw_tsomaxsegcount = scctx->isc_tx_tso_segments_max;
3784 	ifp->if_hw_tsomax = scctx->isc_tx_tso_size_max;
3785 	ifp->if_hw_tsomaxsegsize = scctx->isc_tx_tso_segsize_max;
3786 	if (scctx->isc_rss_table_size == 0)
3787 		scctx->isc_rss_table_size = 64;
3788 	scctx->isc_rss_table_mask = scctx->isc_rss_table_size-1;
3789 
3790 	GROUPTASK_INIT(&ctx->ifc_admin_task, 0, _task_fn_admin, ctx);
3791 	/* XXX format name */
3792 	taskqgroup_attach(qgroup_if_config_tqg, &ctx->ifc_admin_task, ctx, -1, "admin");
3793 	/*
3794 	** Now setup MSI or MSI/X, should
3795 	** return us the number of supported
3796 	** vectors. (Will be 1 for MSI)
3797 	*/
3798 	if (sctx->isc_flags & IFLIB_SKIP_MSIX) {
3799 		msix = scctx->isc_vectors;
3800 	} else if (scctx->isc_msix_bar != 0)
3801 	       /*
3802 		* The simple fact that isc_msix_bar is not 0 does not mean we
3803 		* we have a good value there that is known to work.
3804 		*/
3805 		msix = iflib_msix_init(ctx);
3806 	else {
3807 		scctx->isc_vectors = 1;
3808 		scctx->isc_ntxqsets = 1;
3809 		scctx->isc_nrxqsets = 1;
3810 		scctx->isc_intr = IFLIB_INTR_LEGACY;
3811 		msix = 0;
3812 	}
3813 	/* Get memory for the station queues */
3814 	if ((err = iflib_queues_alloc(ctx))) {
3815 		device_printf(dev, "Unable to allocate queue memory\n");
3816 		goto fail;
3817 	}
3818 
3819 	if ((err = iflib_qset_structures_setup(ctx))) {
3820 		device_printf(dev, "qset structure setup failed %d\n", err);
3821 		goto fail_queues;
3822 	}
3823 
3824 	/*
3825 	 * Group taskqueues aren't properly set up until SMP is started,
3826 	 * so we disable interrupts until we can handle them post
3827 	 * SI_SUB_SMP.
3828 	 *
3829 	 * XXX: disabling interrupts doesn't actually work, at least for
3830 	 * the non-MSI case.  When they occur before SI_SUB_SMP completes,
3831 	 * we do null handling and depend on this not causing too large an
3832 	 * interrupt storm.
3833 	 */
3834 	IFDI_INTR_DISABLE(ctx);
3835 	if (msix > 1 && (err = IFDI_MSIX_INTR_ASSIGN(ctx, msix)) != 0) {
3836 		device_printf(dev, "IFDI_MSIX_INTR_ASSIGN failed %d\n", err);
3837 		goto fail_intr_free;
3838 	}
3839 	if (msix <= 1) {
3840 		rid = 0;
3841 		if (scctx->isc_intr == IFLIB_INTR_MSI) {
3842 			MPASS(msix == 1);
3843 			rid = 1;
3844 		}
3845 		if ((err = iflib_legacy_setup(ctx, ctx->isc_legacy_intr, ctx->ifc_softc, &rid, "irq0")) != 0) {
3846 			device_printf(dev, "iflib_legacy_setup failed %d\n", err);
3847 			goto fail_intr_free;
3848 		}
3849 	}
3850 	ether_ifattach(ctx->ifc_ifp, ctx->ifc_mac);
3851 	if ((err = IFDI_ATTACH_POST(ctx)) != 0) {
3852 		device_printf(dev, "IFDI_ATTACH_POST failed %d\n", err);
3853 		goto fail_detach;
3854 	}
3855 	if ((err = iflib_netmap_attach(ctx))) {
3856 		device_printf(ctx->ifc_dev, "netmap attach failed: %d\n", err);
3857 		goto fail_detach;
3858 	}
3859 	*ctxp = ctx;
3860 
3861 	if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter);
3862 	iflib_add_device_sysctl_post(ctx);
3863 	ctx->ifc_flags |= IFC_INIT_DONE;
3864 	return (0);
3865 fail_detach:
3866 	ether_ifdetach(ctx->ifc_ifp);
3867 fail_intr_free:
3868 	if (scctx->isc_intr == IFLIB_INTR_MSIX || scctx->isc_intr == IFLIB_INTR_MSI)
3869 		pci_release_msi(ctx->ifc_dev);
3870 fail_queues:
3871 	/* XXX free queues */
3872 fail:
3873 	IFDI_DETACH(ctx);
3874 	return (err);
3875 }
3876 
3877 int
3878 iflib_device_attach(device_t dev)
3879 {
3880 	if_ctx_t ctx;
3881 	if_shared_ctx_t sctx;
3882 
3883 	if ((sctx = DEVICE_REGISTER(dev)) == NULL || sctx->isc_magic != IFLIB_MAGIC)
3884 		return (ENOTSUP);
3885 
3886 	pci_enable_busmaster(dev);
3887 
3888 	return (iflib_device_register(dev, NULL, sctx, &ctx));
3889 }
3890 
3891 int
3892 iflib_device_deregister(if_ctx_t ctx)
3893 {
3894 	if_t ifp = ctx->ifc_ifp;
3895 	iflib_txq_t txq;
3896 	iflib_rxq_t rxq;
3897 	device_t dev = ctx->ifc_dev;
3898 	int i;
3899 	struct taskqgroup *tqg;
3900 
3901 	/* Make sure VLANS are not using driver */
3902 	if (if_vlantrunkinuse(ifp)) {
3903 		device_printf(dev,"Vlan in use, detach first\n");
3904 		return (EBUSY);
3905 	}
3906 
3907 	CTX_LOCK(ctx);
3908 	ctx->ifc_in_detach = 1;
3909 	iflib_stop(ctx);
3910 	CTX_UNLOCK(ctx);
3911 
3912 	/* Unregister VLAN events */
3913 	if (ctx->ifc_vlan_attach_event != NULL)
3914 		EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event);
3915 	if (ctx->ifc_vlan_detach_event != NULL)
3916 		EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event);
3917 
3918 	iflib_netmap_detach(ifp);
3919 	ether_ifdetach(ifp);
3920 	/* ether_ifdetach calls if_qflush - lock must be destroy afterwards*/
3921 	CTX_LOCK_DESTROY(ctx);
3922 	if (ctx->ifc_led_dev != NULL)
3923 		led_destroy(ctx->ifc_led_dev);
3924 	/* XXX drain any dependent tasks */
3925 	tqg = qgroup_if_io_tqg;
3926 	for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) {
3927 		callout_drain(&txq->ift_timer);
3928 		callout_drain(&txq->ift_db_check);
3929 		if (txq->ift_task.gt_uniq != NULL)
3930 			taskqgroup_detach(tqg, &txq->ift_task);
3931 	}
3932 	for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) {
3933 		if (rxq->ifr_task.gt_uniq != NULL)
3934 			taskqgroup_detach(tqg, &rxq->ifr_task);
3935 	}
3936 	tqg = qgroup_if_config_tqg;
3937 	if (ctx->ifc_admin_task.gt_uniq != NULL)
3938 		taskqgroup_detach(tqg, &ctx->ifc_admin_task);
3939 	if (ctx->ifc_vflr_task.gt_uniq != NULL)
3940 		taskqgroup_detach(tqg, &ctx->ifc_vflr_task);
3941 
3942 	IFDI_DETACH(ctx);
3943 	device_set_softc(ctx->ifc_dev, NULL);
3944 	if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_LEGACY) {
3945 		pci_release_msi(dev);
3946 	}
3947 	if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_MSIX) {
3948 		iflib_irq_free(ctx, &ctx->ifc_legacy_irq);
3949 	}
3950 	if (ctx->ifc_msix_mem != NULL) {
3951 		bus_release_resource(ctx->ifc_dev, SYS_RES_MEMORY,
3952 			ctx->ifc_softc_ctx.isc_msix_bar, ctx->ifc_msix_mem);
3953 		ctx->ifc_msix_mem = NULL;
3954 	}
3955 
3956 	bus_generic_detach(dev);
3957 	if_free(ifp);
3958 
3959 	iflib_tx_structures_free(ctx);
3960 	iflib_rx_structures_free(ctx);
3961 	if (ctx->ifc_flags & IFC_SC_ALLOCATED)
3962 		free(ctx->ifc_softc, M_IFLIB);
3963 	free(ctx, M_IFLIB);
3964 	return (0);
3965 }
3966 
3967 
3968 int
3969 iflib_device_detach(device_t dev)
3970 {
3971 	if_ctx_t ctx = device_get_softc(dev);
3972 
3973 	return (iflib_device_deregister(ctx));
3974 }
3975 
3976 int
3977 iflib_device_suspend(device_t dev)
3978 {
3979 	if_ctx_t ctx = device_get_softc(dev);
3980 
3981 	CTX_LOCK(ctx);
3982 	IFDI_SUSPEND(ctx);
3983 	CTX_UNLOCK(ctx);
3984 
3985 	return bus_generic_suspend(dev);
3986 }
3987 int
3988 iflib_device_shutdown(device_t dev)
3989 {
3990 	if_ctx_t ctx = device_get_softc(dev);
3991 
3992 	CTX_LOCK(ctx);
3993 	IFDI_SHUTDOWN(ctx);
3994 	CTX_UNLOCK(ctx);
3995 
3996 	return bus_generic_suspend(dev);
3997 }
3998 
3999 
4000 int
4001 iflib_device_resume(device_t dev)
4002 {
4003 	if_ctx_t ctx = device_get_softc(dev);
4004 	iflib_txq_t txq = ctx->ifc_txqs;
4005 
4006 	CTX_LOCK(ctx);
4007 	IFDI_RESUME(ctx);
4008 	iflib_init_locked(ctx);
4009 	CTX_UNLOCK(ctx);
4010 	for (int i = 0; i < NTXQSETS(ctx); i++, txq++)
4011 		iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET);
4012 
4013 	return (bus_generic_resume(dev));
4014 }
4015 
4016 int
4017 iflib_device_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params)
4018 {
4019 	int error;
4020 	if_ctx_t ctx = device_get_softc(dev);
4021 
4022 	CTX_LOCK(ctx);
4023 	error = IFDI_IOV_INIT(ctx, num_vfs, params);
4024 	CTX_UNLOCK(ctx);
4025 
4026 	return (error);
4027 }
4028 
4029 void
4030 iflib_device_iov_uninit(device_t dev)
4031 {
4032 	if_ctx_t ctx = device_get_softc(dev);
4033 
4034 	CTX_LOCK(ctx);
4035 	IFDI_IOV_UNINIT(ctx);
4036 	CTX_UNLOCK(ctx);
4037 }
4038 
4039 int
4040 iflib_device_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params)
4041 {
4042 	int error;
4043 	if_ctx_t ctx = device_get_softc(dev);
4044 
4045 	CTX_LOCK(ctx);
4046 	error = IFDI_IOV_VF_ADD(ctx, vfnum, params);
4047 	CTX_UNLOCK(ctx);
4048 
4049 	return (error);
4050 }
4051 
4052 /*********************************************************************
4053  *
4054  *  MODULE FUNCTION DEFINITIONS
4055  *
4056  **********************************************************************/
4057 
4058 /*
4059  * - Start a fast taskqueue thread for each core
4060  * - Start a taskqueue for control operations
4061  */
4062 static int
4063 iflib_module_init(void)
4064 {
4065 	return (0);
4066 }
4067 
4068 static int
4069 iflib_module_event_handler(module_t mod, int what, void *arg)
4070 {
4071 	int err;
4072 
4073 	switch (what) {
4074 	case MOD_LOAD:
4075 		if ((err = iflib_module_init()) != 0)
4076 			return (err);
4077 		break;
4078 	case MOD_UNLOAD:
4079 		return (EBUSY);
4080 	default:
4081 		return (EOPNOTSUPP);
4082 	}
4083 
4084 	return (0);
4085 }
4086 
4087 /*********************************************************************
4088  *
4089  *  PUBLIC FUNCTION DEFINITIONS
4090  *     ordered as in iflib.h
4091  *
4092  **********************************************************************/
4093 
4094 
4095 static void
4096 _iflib_assert(if_shared_ctx_t sctx)
4097 {
4098 	MPASS(sctx->isc_tx_maxsize);
4099 	MPASS(sctx->isc_tx_maxsegsize);
4100 
4101 	MPASS(sctx->isc_rx_maxsize);
4102 	MPASS(sctx->isc_rx_nsegments);
4103 	MPASS(sctx->isc_rx_maxsegsize);
4104 
4105 	MPASS(sctx->isc_nrxd_min[0]);
4106 	MPASS(sctx->isc_nrxd_max[0]);
4107 	MPASS(sctx->isc_nrxd_default[0]);
4108 	MPASS(sctx->isc_ntxd_min[0]);
4109 	MPASS(sctx->isc_ntxd_max[0]);
4110 	MPASS(sctx->isc_ntxd_default[0]);
4111 }
4112 
4113 static void
4114 _iflib_pre_assert(if_softc_ctx_t scctx)
4115 {
4116 
4117 	MPASS(scctx->isc_txrx->ift_txd_encap);
4118 	MPASS(scctx->isc_txrx->ift_txd_flush);
4119 	MPASS(scctx->isc_txrx->ift_txd_credits_update);
4120 	MPASS(scctx->isc_txrx->ift_rxd_available);
4121 	MPASS(scctx->isc_txrx->ift_rxd_pkt_get);
4122 	MPASS(scctx->isc_txrx->ift_rxd_refill);
4123 	MPASS(scctx->isc_txrx->ift_rxd_flush);
4124 }
4125 
4126 static int
4127 iflib_register(if_ctx_t ctx)
4128 {
4129 	if_shared_ctx_t sctx = ctx->ifc_sctx;
4130 	driver_t *driver = sctx->isc_driver;
4131 	device_t dev = ctx->ifc_dev;
4132 	if_t ifp;
4133 
4134 	_iflib_assert(sctx);
4135 
4136 	CTX_LOCK_INIT(ctx, device_get_nameunit(ctx->ifc_dev));
4137 
4138 	ifp = ctx->ifc_ifp = if_gethandle(IFT_ETHER);
4139 	if (ifp == NULL) {
4140 		device_printf(dev, "can not allocate ifnet structure\n");
4141 		return (ENOMEM);
4142 	}
4143 
4144 	/*
4145 	 * Initialize our context's device specific methods
4146 	 */
4147 	kobj_init((kobj_t) ctx, (kobj_class_t) driver);
4148 	kobj_class_compile((kobj_class_t) driver);
4149 	driver->refs++;
4150 
4151 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
4152 	if_setsoftc(ifp, ctx);
4153 	if_setdev(ifp, dev);
4154 	if_setinitfn(ifp, iflib_if_init);
4155 	if_setioctlfn(ifp, iflib_if_ioctl);
4156 	if_settransmitfn(ifp, iflib_if_transmit);
4157 	if_setqflushfn(ifp, iflib_if_qflush);
4158 	if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
4159 
4160 	ctx->ifc_vlan_attach_event =
4161 		EVENTHANDLER_REGISTER(vlan_config, iflib_vlan_register, ctx,
4162 							  EVENTHANDLER_PRI_FIRST);
4163 	ctx->ifc_vlan_detach_event =
4164 		EVENTHANDLER_REGISTER(vlan_unconfig, iflib_vlan_unregister, ctx,
4165 							  EVENTHANDLER_PRI_FIRST);
4166 
4167 	ifmedia_init(&ctx->ifc_media, IFM_IMASK,
4168 					 iflib_media_change, iflib_media_status);
4169 
4170 	return (0);
4171 }
4172 
4173 
4174 static int
4175 iflib_queues_alloc(if_ctx_t ctx)
4176 {
4177 	if_shared_ctx_t sctx = ctx->ifc_sctx;
4178 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
4179 	device_t dev = ctx->ifc_dev;
4180 	int nrxqsets = scctx->isc_nrxqsets;
4181 	int ntxqsets = scctx->isc_ntxqsets;
4182 	iflib_txq_t txq;
4183 	iflib_rxq_t rxq;
4184 	iflib_fl_t fl = NULL;
4185 	int i, j, cpu, err, txconf, rxconf;
4186 	iflib_dma_info_t ifdip;
4187 	uint32_t *rxqsizes = scctx->isc_rxqsizes;
4188 	uint32_t *txqsizes = scctx->isc_txqsizes;
4189 	uint8_t nrxqs = sctx->isc_nrxqs;
4190 	uint8_t ntxqs = sctx->isc_ntxqs;
4191 	int nfree_lists = sctx->isc_nfl ? sctx->isc_nfl : 1;
4192 	caddr_t *vaddrs;
4193 	uint64_t *paddrs;
4194 	struct ifmp_ring **brscp;
4195 	int nbuf_rings = 1; /* XXX determine dynamically */
4196 
4197 	KASSERT(ntxqs > 0, ("number of queues per qset must be at least 1"));
4198 	KASSERT(nrxqs > 0, ("number of queues per qset must be at least 1"));
4199 
4200 	brscp = NULL;
4201 	txq = NULL;
4202 	rxq = NULL;
4203 
4204 /* Allocate the TX ring struct memory */
4205 	if (!(txq =
4206 	    (iflib_txq_t) malloc(sizeof(struct iflib_txq) *
4207 	    ntxqsets, M_IFLIB, M_NOWAIT | M_ZERO))) {
4208 		device_printf(dev, "Unable to allocate TX ring memory\n");
4209 		err = ENOMEM;
4210 		goto fail;
4211 	}
4212 
4213 	/* Now allocate the RX */
4214 	if (!(rxq =
4215 	    (iflib_rxq_t) malloc(sizeof(struct iflib_rxq) *
4216 	    nrxqsets, M_IFLIB, M_NOWAIT | M_ZERO))) {
4217 		device_printf(dev, "Unable to allocate RX ring memory\n");
4218 		err = ENOMEM;
4219 		goto rx_fail;
4220 	}
4221 	if (!(brscp = malloc(sizeof(void *) * nbuf_rings * nrxqsets, M_IFLIB, M_NOWAIT | M_ZERO))) {
4222 		device_printf(dev, "Unable to buf_ring_sc * memory\n");
4223 		err = ENOMEM;
4224 		goto rx_fail;
4225 	}
4226 
4227 	ctx->ifc_txqs = txq;
4228 	ctx->ifc_rxqs = rxq;
4229 
4230 	/*
4231 	 * XXX handle allocation failure
4232 	 */
4233 	for (txconf = i = 0, cpu = CPU_FIRST(); i < ntxqsets; i++, txconf++, txq++, cpu = CPU_NEXT(cpu)) {
4234 		/* Set up some basics */
4235 
4236 		if ((ifdip = malloc(sizeof(struct iflib_dma_info) * ntxqs, M_IFLIB, M_WAITOK|M_ZERO)) == NULL) {
4237 			device_printf(dev, "failed to allocate iflib_dma_info\n");
4238 			err = ENOMEM;
4239 			goto err_tx_desc;
4240 		}
4241 		txq->ift_ifdi = ifdip;
4242 		for (j = 0; j < ntxqs; j++, ifdip++) {
4243 			if (iflib_dma_alloc(ctx, txqsizes[j], ifdip, BUS_DMA_NOWAIT)) {
4244 				device_printf(dev, "Unable to allocate Descriptor memory\n");
4245 				err = ENOMEM;
4246 				goto err_tx_desc;
4247 			}
4248 			bzero((void *)ifdip->idi_vaddr, txqsizes[j]);
4249 		}
4250 		txq->ift_ctx = ctx;
4251 		txq->ift_id = i;
4252 		if (sctx->isc_flags & IFLIB_HAS_TXCQ) {
4253 			txq->ift_br_offset = 1;
4254 		} else {
4255 			txq->ift_br_offset = 0;
4256 		}
4257 		/* XXX fix this */
4258 		txq->ift_timer.c_cpu = cpu;
4259 		txq->ift_db_check.c_cpu = cpu;
4260 		txq->ift_nbr = nbuf_rings;
4261 
4262 		if (iflib_txsd_alloc(txq)) {
4263 			device_printf(dev, "Critical Failure setting up TX buffers\n");
4264 			err = ENOMEM;
4265 			goto err_tx_desc;
4266 		}
4267 
4268 		/* Initialize the TX lock */
4269 		snprintf(txq->ift_mtx_name, MTX_NAME_LEN, "%s:tx(%d):callout",
4270 		    device_get_nameunit(dev), txq->ift_id);
4271 		mtx_init(&txq->ift_mtx, txq->ift_mtx_name, NULL, MTX_DEF);
4272 		callout_init_mtx(&txq->ift_timer, &txq->ift_mtx, 0);
4273 		callout_init_mtx(&txq->ift_db_check, &txq->ift_mtx, 0);
4274 
4275 		snprintf(txq->ift_db_mtx_name, MTX_NAME_LEN, "%s:tx(%d):db",
4276 			 device_get_nameunit(dev), txq->ift_id);
4277 		TXDB_LOCK_INIT(txq);
4278 
4279 		txq->ift_br = brscp + i*nbuf_rings;
4280 		for (j = 0; j < nbuf_rings; j++) {
4281 			err = ifmp_ring_alloc(&txq->ift_br[j], 2048, txq, iflib_txq_drain,
4282 					      iflib_txq_can_drain, M_IFLIB, M_WAITOK);
4283 			if (err) {
4284 				/* XXX free any allocated rings */
4285 				device_printf(dev, "Unable to allocate buf_ring\n");
4286 				goto err_tx_desc;
4287 			}
4288 		}
4289 	}
4290 
4291 	for (rxconf = i = 0; i < nrxqsets; i++, rxconf++, rxq++) {
4292 		/* Set up some basics */
4293 
4294 		if ((ifdip = malloc(sizeof(struct iflib_dma_info) * nrxqs, M_IFLIB, M_WAITOK|M_ZERO)) == NULL) {
4295 			device_printf(dev, "failed to allocate iflib_dma_info\n");
4296 			err = ENOMEM;
4297 			goto err_tx_desc;
4298 		}
4299 
4300 		rxq->ifr_ifdi = ifdip;
4301 		for (j = 0; j < nrxqs; j++, ifdip++) {
4302 			if (iflib_dma_alloc(ctx, rxqsizes[j], ifdip, BUS_DMA_NOWAIT)) {
4303 				device_printf(dev, "Unable to allocate Descriptor memory\n");
4304 				err = ENOMEM;
4305 				goto err_tx_desc;
4306 			}
4307 			bzero((void *)ifdip->idi_vaddr, rxqsizes[j]);
4308 		}
4309 		rxq->ifr_ctx = ctx;
4310 		rxq->ifr_id = i;
4311 		if (sctx->isc_flags & IFLIB_HAS_RXCQ) {
4312 			rxq->ifr_fl_offset = 1;
4313 		} else {
4314 			rxq->ifr_fl_offset = 0;
4315 		}
4316 		rxq->ifr_nfl = nfree_lists;
4317 		if (!(fl =
4318 			  (iflib_fl_t) malloc(sizeof(struct iflib_fl) * nfree_lists, M_IFLIB, M_NOWAIT | M_ZERO))) {
4319 			device_printf(dev, "Unable to allocate free list memory\n");
4320 			err = ENOMEM;
4321 			goto err_tx_desc;
4322 		}
4323 		rxq->ifr_fl = fl;
4324 		for (j = 0; j < nfree_lists; j++) {
4325 			rxq->ifr_fl[j].ifl_rxq = rxq;
4326 			rxq->ifr_fl[j].ifl_id = j;
4327 			rxq->ifr_fl[j].ifl_ifdi =
4328 			    &rxq->ifr_ifdi[j + rxq->ifr_fl_offset];
4329 		}
4330         /* Allocate receive buffers for the ring*/
4331 		if (iflib_rxsd_alloc(rxq)) {
4332 			device_printf(dev,
4333 			    "Critical Failure setting up receive buffers\n");
4334 			err = ENOMEM;
4335 			goto err_rx_desc;
4336 		}
4337 	}
4338 
4339 	/* TXQs */
4340 	vaddrs = malloc(sizeof(caddr_t)*ntxqsets*ntxqs, M_IFLIB, M_WAITOK);
4341 	paddrs = malloc(sizeof(uint64_t)*ntxqsets*ntxqs, M_IFLIB, M_WAITOK);
4342 	for (i = 0; i < ntxqsets; i++) {
4343 		iflib_dma_info_t di = ctx->ifc_txqs[i].ift_ifdi;
4344 
4345 		for (j = 0; j < ntxqs; j++, di++) {
4346 			vaddrs[i*ntxqs + j] = di->idi_vaddr;
4347 			paddrs[i*ntxqs + j] = di->idi_paddr;
4348 		}
4349 	}
4350 	if ((err = IFDI_TX_QUEUES_ALLOC(ctx, vaddrs, paddrs, ntxqs, ntxqsets)) != 0) {
4351 		device_printf(ctx->ifc_dev, "device queue allocation failed\n");
4352 		iflib_tx_structures_free(ctx);
4353 		free(vaddrs, M_IFLIB);
4354 		free(paddrs, M_IFLIB);
4355 		goto err_rx_desc;
4356 	}
4357 	free(vaddrs, M_IFLIB);
4358 	free(paddrs, M_IFLIB);
4359 
4360 	/* RXQs */
4361 	vaddrs = malloc(sizeof(caddr_t)*nrxqsets*nrxqs, M_IFLIB, M_WAITOK);
4362 	paddrs = malloc(sizeof(uint64_t)*nrxqsets*nrxqs, M_IFLIB, M_WAITOK);
4363 	for (i = 0; i < nrxqsets; i++) {
4364 		iflib_dma_info_t di = ctx->ifc_rxqs[i].ifr_ifdi;
4365 
4366 		for (j = 0; j < nrxqs; j++, di++) {
4367 			vaddrs[i*nrxqs + j] = di->idi_vaddr;
4368 			paddrs[i*nrxqs + j] = di->idi_paddr;
4369 		}
4370 	}
4371 	if ((err = IFDI_RX_QUEUES_ALLOC(ctx, vaddrs, paddrs, nrxqs, nrxqsets)) != 0) {
4372 		device_printf(ctx->ifc_dev, "device queue allocation failed\n");
4373 		iflib_tx_structures_free(ctx);
4374 		free(vaddrs, M_IFLIB);
4375 		free(paddrs, M_IFLIB);
4376 		goto err_rx_desc;
4377 	}
4378 	free(vaddrs, M_IFLIB);
4379 	free(paddrs, M_IFLIB);
4380 
4381 	return (0);
4382 
4383 /* XXX handle allocation failure changes */
4384 err_rx_desc:
4385 err_tx_desc:
4386 	if (ctx->ifc_rxqs != NULL)
4387 		free(ctx->ifc_rxqs, M_IFLIB);
4388 	ctx->ifc_rxqs = NULL;
4389 	if (ctx->ifc_txqs != NULL)
4390 		free(ctx->ifc_txqs, M_IFLIB);
4391 	ctx->ifc_txqs = NULL;
4392 rx_fail:
4393 	if (brscp != NULL)
4394 		free(brscp, M_IFLIB);
4395 	if (rxq != NULL)
4396 		free(rxq, M_IFLIB);
4397 	if (txq != NULL)
4398 		free(txq, M_IFLIB);
4399 fail:
4400 	return (err);
4401 }
4402 
4403 static int
4404 iflib_tx_structures_setup(if_ctx_t ctx)
4405 {
4406 	iflib_txq_t txq = ctx->ifc_txqs;
4407 	int i;
4408 
4409 	for (i = 0; i < NTXQSETS(ctx); i++, txq++)
4410 		iflib_txq_setup(txq);
4411 
4412 	return (0);
4413 }
4414 
4415 static void
4416 iflib_tx_structures_free(if_ctx_t ctx)
4417 {
4418 	iflib_txq_t txq = ctx->ifc_txqs;
4419 	int i, j;
4420 
4421 	for (i = 0; i < NTXQSETS(ctx); i++, txq++) {
4422 		iflib_txq_destroy(txq);
4423 		for (j = 0; j < ctx->ifc_nhwtxqs; j++)
4424 			iflib_dma_free(&txq->ift_ifdi[j]);
4425 	}
4426 	free(ctx->ifc_txqs, M_IFLIB);
4427 	ctx->ifc_txqs = NULL;
4428 	IFDI_QUEUES_FREE(ctx);
4429 }
4430 
4431 /*********************************************************************
4432  *
4433  *  Initialize all receive rings.
4434  *
4435  **********************************************************************/
4436 static int
4437 iflib_rx_structures_setup(if_ctx_t ctx)
4438 {
4439 	iflib_rxq_t rxq = ctx->ifc_rxqs;
4440 	int q;
4441 #if defined(INET6) || defined(INET)
4442 	int i, err;
4443 #endif
4444 
4445 	for (q = 0; q < ctx->ifc_softc_ctx.isc_nrxqsets; q++, rxq++) {
4446 #if defined(INET6) || defined(INET)
4447 		tcp_lro_free(&rxq->ifr_lc);
4448 		if ((err = tcp_lro_init_args(&rxq->ifr_lc, ctx->ifc_ifp,
4449 		    TCP_LRO_ENTRIES, min(1024,
4450 		    ctx->ifc_softc_ctx.isc_nrxd[rxq->ifr_fl_offset]))) != 0) {
4451 			device_printf(ctx->ifc_dev, "LRO Initialization failed!\n");
4452 			goto fail;
4453 		}
4454 		rxq->ifr_lro_enabled = TRUE;
4455 #endif
4456 		IFDI_RXQ_SETUP(ctx, rxq->ifr_id);
4457 	}
4458 	return (0);
4459 #if defined(INET6) || defined(INET)
4460 fail:
4461 	/*
4462 	 * Free RX software descriptors allocated so far, we will only handle
4463 	 * the rings that completed, the failing case will have
4464 	 * cleaned up for itself. 'q' failed, so its the terminus.
4465 	 */
4466 	rxq = ctx->ifc_rxqs;
4467 	for (i = 0; i < q; ++i, rxq++) {
4468 		iflib_rx_sds_free(rxq);
4469 		rxq->ifr_cq_gen = rxq->ifr_cq_cidx = rxq->ifr_cq_pidx = 0;
4470 	}
4471 	return (err);
4472 #endif
4473 }
4474 
4475 /*********************************************************************
4476  *
4477  *  Free all receive rings.
4478  *
4479  **********************************************************************/
4480 static void
4481 iflib_rx_structures_free(if_ctx_t ctx)
4482 {
4483 	iflib_rxq_t rxq = ctx->ifc_rxqs;
4484 
4485 	for (int i = 0; i < ctx->ifc_softc_ctx.isc_nrxqsets; i++, rxq++) {
4486 		iflib_rx_sds_free(rxq);
4487 	}
4488 }
4489 
4490 static int
4491 iflib_qset_structures_setup(if_ctx_t ctx)
4492 {
4493 	int err;
4494 
4495 	if ((err = iflib_tx_structures_setup(ctx)) != 0)
4496 		return (err);
4497 
4498 	if ((err = iflib_rx_structures_setup(ctx)) != 0) {
4499 		device_printf(ctx->ifc_dev, "iflib_rx_structures_setup failed: %d\n", err);
4500 		iflib_tx_structures_free(ctx);
4501 		iflib_rx_structures_free(ctx);
4502 	}
4503 	return (err);
4504 }
4505 
4506 int
4507 iflib_irq_alloc(if_ctx_t ctx, if_irq_t irq, int rid,
4508 				driver_filter_t filter, void *filter_arg, driver_intr_t handler, void *arg, char *name)
4509 {
4510 
4511 	return (_iflib_irq_alloc(ctx, irq, rid, filter, handler, arg, name));
4512 }
4513 
4514 static int
4515 find_nth(if_ctx_t ctx, cpuset_t *cpus, int qid)
4516 {
4517 	int i, cpuid, eqid, count;
4518 
4519 	CPU_COPY(&ctx->ifc_cpus, cpus);
4520 	count = CPU_COUNT(&ctx->ifc_cpus);
4521 	eqid = qid % count;
4522 	/* clear up to the qid'th bit */
4523 	for (i = 0; i < eqid; i++) {
4524 		cpuid = CPU_FFS(cpus);
4525 		MPASS(cpuid != 0);
4526 		CPU_CLR(cpuid-1, cpus);
4527 	}
4528 	cpuid = CPU_FFS(cpus);
4529 	MPASS(cpuid != 0);
4530 	return (cpuid-1);
4531 }
4532 
4533 int
4534 iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid,
4535 						iflib_intr_type_t type, driver_filter_t *filter,
4536 						void *filter_arg, int qid, char *name)
4537 {
4538 	struct grouptask *gtask;
4539 	struct taskqgroup *tqg;
4540 	iflib_filter_info_t info;
4541 	cpuset_t cpus;
4542 	gtask_fn_t *fn;
4543 	int tqrid, err, cpuid;
4544 	void *q;
4545 
4546 	info = &ctx->ifc_filter_info;
4547 	tqrid = rid;
4548 
4549 	switch (type) {
4550 	/* XXX merge tx/rx for netmap? */
4551 	case IFLIB_INTR_TX:
4552 		q = &ctx->ifc_txqs[qid];
4553 		info = &ctx->ifc_txqs[qid].ift_filter_info;
4554 		gtask = &ctx->ifc_txqs[qid].ift_task;
4555 		tqg = qgroup_if_io_tqg;
4556 		fn = _task_fn_tx;
4557 		GROUPTASK_INIT(gtask, 0, fn, q);
4558 		break;
4559 	case IFLIB_INTR_RX:
4560 		q = &ctx->ifc_rxqs[qid];
4561 		info = &ctx->ifc_rxqs[qid].ifr_filter_info;
4562 		gtask = &ctx->ifc_rxqs[qid].ifr_task;
4563 		tqg = qgroup_if_io_tqg;
4564 		fn = _task_fn_rx;
4565 		GROUPTASK_INIT(gtask, 0, fn, q);
4566 		break;
4567 	case IFLIB_INTR_ADMIN:
4568 		q = ctx;
4569 		tqrid = -1;
4570 		info = &ctx->ifc_filter_info;
4571 		gtask = &ctx->ifc_admin_task;
4572 		tqg = qgroup_if_config_tqg;
4573 		fn = _task_fn_admin;
4574 		break;
4575 	default:
4576 		panic("unknown net intr type");
4577 	}
4578 
4579 	info->ifi_filter = filter;
4580 	info->ifi_filter_arg = filter_arg;
4581 	info->ifi_task = gtask;
4582 	info->ifi_ctx = ctx;
4583 
4584 	err = _iflib_irq_alloc(ctx, irq, rid, iflib_fast_intr, NULL, info,  name);
4585 	if (err != 0) {
4586 		device_printf(ctx->ifc_dev, "_iflib_irq_alloc failed %d\n", err);
4587 		return (err);
4588 	}
4589 	if (type == IFLIB_INTR_ADMIN)
4590 		return (0);
4591 
4592 	if (tqrid != -1) {
4593 		cpuid = find_nth(ctx, &cpus, qid);
4594 		taskqgroup_attach_cpu(tqg, gtask, q, cpuid, irq->ii_rid, name);
4595 	} else {
4596 		taskqgroup_attach(tqg, gtask, q, tqrid, name);
4597 	}
4598 
4599 	return (0);
4600 }
4601 
4602 void
4603 iflib_softirq_alloc_generic(if_ctx_t ctx, int rid, iflib_intr_type_t type,  void *arg, int qid, char *name)
4604 {
4605 	struct grouptask *gtask;
4606 	struct taskqgroup *tqg;
4607 	gtask_fn_t *fn;
4608 	void *q;
4609 
4610 	switch (type) {
4611 	case IFLIB_INTR_TX:
4612 		q = &ctx->ifc_txqs[qid];
4613 		gtask = &ctx->ifc_txqs[qid].ift_task;
4614 		tqg = qgroup_if_io_tqg;
4615 		fn = _task_fn_tx;
4616 		break;
4617 	case IFLIB_INTR_RX:
4618 		q = &ctx->ifc_rxqs[qid];
4619 		gtask = &ctx->ifc_rxqs[qid].ifr_task;
4620 		tqg = qgroup_if_io_tqg;
4621 		fn = _task_fn_rx;
4622 		break;
4623 	case IFLIB_INTR_IOV:
4624 		q = ctx;
4625 		gtask = &ctx->ifc_vflr_task;
4626 		tqg = qgroup_if_config_tqg;
4627 		rid = -1;
4628 		fn = _task_fn_iov;
4629 		break;
4630 	default:
4631 		panic("unknown net intr type");
4632 	}
4633 	GROUPTASK_INIT(gtask, 0, fn, q);
4634 	taskqgroup_attach(tqg, gtask, q, rid, name);
4635 }
4636 
4637 void
4638 iflib_irq_free(if_ctx_t ctx, if_irq_t irq)
4639 {
4640 	if (irq->ii_tag)
4641 		bus_teardown_intr(ctx->ifc_dev, irq->ii_res, irq->ii_tag);
4642 
4643 	if (irq->ii_res)
4644 		bus_release_resource(ctx->ifc_dev, SYS_RES_IRQ, irq->ii_rid, irq->ii_res);
4645 }
4646 
4647 static int
4648 iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filter_arg, int *rid, char *name)
4649 {
4650 	iflib_txq_t txq = ctx->ifc_txqs;
4651 	iflib_rxq_t rxq = ctx->ifc_rxqs;
4652 	if_irq_t irq = &ctx->ifc_legacy_irq;
4653 	iflib_filter_info_t info;
4654 	struct grouptask *gtask;
4655 	struct taskqgroup *tqg;
4656 	gtask_fn_t *fn;
4657 	int tqrid;
4658 	void *q;
4659 	int err;
4660 
4661 	q = &ctx->ifc_rxqs[0];
4662 	info = &rxq[0].ifr_filter_info;
4663 	gtask = &rxq[0].ifr_task;
4664 	tqg = qgroup_if_io_tqg;
4665 	tqrid = irq->ii_rid = *rid;
4666 	fn = _task_fn_rx;
4667 
4668 	ctx->ifc_flags |= IFC_LEGACY;
4669 	info->ifi_filter = filter;
4670 	info->ifi_filter_arg = filter_arg;
4671 	info->ifi_task = gtask;
4672 	info->ifi_ctx = ctx;
4673 
4674 	/* We allocate a single interrupt resource */
4675 	if ((err = _iflib_irq_alloc(ctx, irq, tqrid, iflib_fast_intr, NULL, info, name)) != 0)
4676 		return (err);
4677 	GROUPTASK_INIT(gtask, 0, fn, q);
4678 	taskqgroup_attach(tqg, gtask, q, tqrid, name);
4679 
4680 	GROUPTASK_INIT(&txq->ift_task, 0, _task_fn_tx, txq);
4681 	taskqgroup_attach(qgroup_if_io_tqg, &txq->ift_task, txq, tqrid, "tx");
4682 	return (0);
4683 }
4684 
4685 void
4686 iflib_led_create(if_ctx_t ctx)
4687 {
4688 
4689 	ctx->ifc_led_dev = led_create(iflib_led_func, ctx,
4690 								  device_get_nameunit(ctx->ifc_dev));
4691 }
4692 
4693 void
4694 iflib_tx_intr_deferred(if_ctx_t ctx, int txqid)
4695 {
4696 
4697 	GROUPTASK_ENQUEUE(&ctx->ifc_txqs[txqid].ift_task);
4698 }
4699 
4700 void
4701 iflib_rx_intr_deferred(if_ctx_t ctx, int rxqid)
4702 {
4703 
4704 	GROUPTASK_ENQUEUE(&ctx->ifc_rxqs[rxqid].ifr_task);
4705 }
4706 
4707 void
4708 iflib_admin_intr_deferred(if_ctx_t ctx)
4709 {
4710 #ifdef INVARIANTS
4711 	struct grouptask *gtask;
4712 
4713 	gtask = &ctx->ifc_admin_task;
4714 	MPASS(gtask->gt_taskqueue != NULL);
4715 #endif
4716 
4717 	GROUPTASK_ENQUEUE(&ctx->ifc_admin_task);
4718 }
4719 
4720 void
4721 iflib_iov_intr_deferred(if_ctx_t ctx)
4722 {
4723 
4724 	GROUPTASK_ENQUEUE(&ctx->ifc_vflr_task);
4725 }
4726 
4727 void
4728 iflib_io_tqg_attach(struct grouptask *gt, void *uniq, int cpu, char *name)
4729 {
4730 
4731 	taskqgroup_attach_cpu(qgroup_if_io_tqg, gt, uniq, cpu, -1, name);
4732 }
4733 
4734 void
4735 iflib_config_gtask_init(if_ctx_t ctx, struct grouptask *gtask, gtask_fn_t *fn,
4736 	char *name)
4737 {
4738 
4739 	GROUPTASK_INIT(gtask, 0, fn, ctx);
4740 	taskqgroup_attach(qgroup_if_config_tqg, gtask, gtask, -1, name);
4741 }
4742 
4743 void
4744 iflib_config_gtask_deinit(struct grouptask *gtask)
4745 {
4746 
4747 	taskqgroup_detach(qgroup_if_config_tqg, gtask);
4748 }
4749 
4750 void
4751 iflib_link_state_change(if_ctx_t ctx, int link_state, uint64_t baudrate)
4752 {
4753 	if_t ifp = ctx->ifc_ifp;
4754 	iflib_txq_t txq = ctx->ifc_txqs;
4755 
4756 	if_setbaudrate(ifp, baudrate);
4757 
4758 	/* If link down, disable watchdog */
4759 	if ((ctx->ifc_link_state == LINK_STATE_UP) && (link_state == LINK_STATE_DOWN)) {
4760 		for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxqsets; i++, txq++)
4761 			txq->ift_qstatus = IFLIB_QUEUE_IDLE;
4762 	}
4763 	ctx->ifc_link_state = link_state;
4764 	if_link_state_change(ifp, link_state);
4765 }
4766 
4767 static int
4768 iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq)
4769 {
4770 	int credits;
4771 #ifdef INVARIANTS
4772 	int credits_pre = txq->ift_cidx_processed;
4773 #endif
4774 
4775 	if (ctx->isc_txd_credits_update == NULL)
4776 		return (0);
4777 
4778 	if ((credits = ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, txq->ift_cidx_processed, true)) == 0)
4779 		return (0);
4780 
4781 	txq->ift_processed += credits;
4782 	txq->ift_cidx_processed += credits;
4783 
4784 	MPASS(credits_pre + credits == txq->ift_cidx_processed);
4785 	if (txq->ift_cidx_processed >= txq->ift_size)
4786 		txq->ift_cidx_processed -= txq->ift_size;
4787 	return (credits);
4788 }
4789 
4790 static int
4791 iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, int cidx, int budget)
4792 {
4793 
4794 	return (ctx->isc_rxd_available(ctx->ifc_softc, rxq->ifr_id, cidx,
4795 	    budget));
4796 }
4797 
4798 void
4799 iflib_add_int_delay_sysctl(if_ctx_t ctx, const char *name,
4800 	const char *description, if_int_delay_info_t info,
4801 	int offset, int value)
4802 {
4803 	info->iidi_ctx = ctx;
4804 	info->iidi_offset = offset;
4805 	info->iidi_value = value;
4806 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(ctx->ifc_dev),
4807 	    SYSCTL_CHILDREN(device_get_sysctl_tree(ctx->ifc_dev)),
4808 	    OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
4809 	    info, 0, iflib_sysctl_int_delay, "I", description);
4810 }
4811 
4812 struct mtx *
4813 iflib_ctx_lock_get(if_ctx_t ctx)
4814 {
4815 
4816 	return (&ctx->ifc_mtx);
4817 }
4818 
4819 static int
4820 iflib_msix_init(if_ctx_t ctx)
4821 {
4822 	device_t dev = ctx->ifc_dev;
4823 	if_shared_ctx_t sctx = ctx->ifc_sctx;
4824 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
4825 	int vectors, queues, rx_queues, tx_queues, queuemsgs, msgs;
4826 	int iflib_num_tx_queues, iflib_num_rx_queues;
4827 	int err, admincnt, bar;
4828 
4829 	iflib_num_tx_queues = scctx->isc_ntxqsets;
4830 	iflib_num_rx_queues = scctx->isc_nrxqsets;
4831 
4832 	device_printf(dev, "msix_init qsets capped at %d\n", iflib_num_tx_queues);
4833 
4834 	bar = ctx->ifc_softc_ctx.isc_msix_bar;
4835 	admincnt = sctx->isc_admin_intrcnt;
4836 	/* Override by tuneable */
4837 	if (enable_msix == 0)
4838 		goto msi;
4839 
4840 	/*
4841 	** When used in a virtualized environment
4842 	** PCI BUSMASTER capability may not be set
4843 	** so explicity set it here and rewrite
4844 	** the ENABLE in the MSIX control register
4845 	** at this point to cause the host to
4846 	** successfully initialize us.
4847 	*/
4848 	{
4849 		int msix_ctrl, rid;
4850 
4851  		pci_enable_busmaster(dev);
4852 		rid = 0;
4853 		if (pci_find_cap(dev, PCIY_MSIX, &rid) == 0 && rid != 0) {
4854 			rid += PCIR_MSIX_CTRL;
4855 			msix_ctrl = pci_read_config(dev, rid, 2);
4856 			msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
4857 			pci_write_config(dev, rid, msix_ctrl, 2);
4858 		} else {
4859 			device_printf(dev, "PCIY_MSIX capability not found; "
4860 			                   "or rid %d == 0.\n", rid);
4861 			goto msi;
4862 		}
4863 	}
4864 
4865 	/*
4866 	 * bar == -1 => "trust me I know what I'm doing"
4867 	 * https://www.youtube.com/watch?v=nnwWKkNau4I
4868 	 * Some drivers are for hardware that is so shoddily
4869 	 * documented that no one knows which bars are which
4870 	 * so the developer has to map all bars. This hack
4871 	 * allows shoddy garbage to use msix in this framework.
4872 	 */
4873 	if (bar != -1) {
4874 		ctx->ifc_msix_mem = bus_alloc_resource_any(dev,
4875 	            SYS_RES_MEMORY, &bar, RF_ACTIVE);
4876 		if (ctx->ifc_msix_mem == NULL) {
4877 			/* May not be enabled */
4878 			device_printf(dev, "Unable to map MSIX table \n");
4879 			goto msi;
4880 		}
4881 	}
4882 	/* First try MSI/X */
4883 	if ((msgs = pci_msix_count(dev)) == 0) { /* system has msix disabled */
4884 		device_printf(dev, "System has MSIX disabled \n");
4885 		bus_release_resource(dev, SYS_RES_MEMORY,
4886 		    bar, ctx->ifc_msix_mem);
4887 		ctx->ifc_msix_mem = NULL;
4888 		goto msi;
4889 	}
4890 #if IFLIB_DEBUG
4891 	/* use only 1 qset in debug mode */
4892 	queuemsgs = min(msgs - admincnt, 1);
4893 #else
4894 	queuemsgs = msgs - admincnt;
4895 #endif
4896 	if (bus_get_cpus(dev, INTR_CPUS, sizeof(ctx->ifc_cpus), &ctx->ifc_cpus) == 0) {
4897 #ifdef RSS
4898 		queues = imin(queuemsgs, rss_getnumbuckets());
4899 #else
4900 		queues = queuemsgs;
4901 #endif
4902 		queues = imin(CPU_COUNT(&ctx->ifc_cpus), queues);
4903 		device_printf(dev, "pxm cpus: %d queue msgs: %d admincnt: %d\n",
4904 					  CPU_COUNT(&ctx->ifc_cpus), queuemsgs, admincnt);
4905 	} else {
4906 		device_printf(dev, "Unable to fetch CPU list\n");
4907 		/* Figure out a reasonable auto config value */
4908 		queues = min(queuemsgs, mp_ncpus);
4909 	}
4910 #ifdef  RSS
4911 	/* If we're doing RSS, clamp at the number of RSS buckets */
4912 	if (queues > rss_getnumbuckets())
4913 		queues = rss_getnumbuckets();
4914 #endif
4915 	if (iflib_num_rx_queues > 0 && iflib_num_rx_queues < queuemsgs - admincnt)
4916 		rx_queues = iflib_num_rx_queues;
4917 	else
4918 		rx_queues = queues;
4919 	/*
4920 	 * We want this to be all logical CPUs by default
4921 	 */
4922 	if (iflib_num_tx_queues > 0 && iflib_num_tx_queues < queues)
4923 		tx_queues = iflib_num_tx_queues;
4924 	else
4925 		tx_queues = mp_ncpus;
4926 
4927 	if (ctx->ifc_sysctl_qs_eq_override == 0) {
4928 #ifdef INVARIANTS
4929 		if (tx_queues != rx_queues)
4930 			device_printf(dev, "queue equality override not set, capping rx_queues at %d and tx_queues at %d\n",
4931 				      min(rx_queues, tx_queues), min(rx_queues, tx_queues));
4932 #endif
4933 		tx_queues = min(rx_queues, tx_queues);
4934 		rx_queues = min(rx_queues, tx_queues);
4935 	}
4936 
4937 	device_printf(dev, "using %d rx queues %d tx queues \n", rx_queues, tx_queues);
4938 
4939 	vectors = rx_queues + admincnt;
4940 	if ((err = pci_alloc_msix(dev, &vectors)) == 0) {
4941 		device_printf(dev,
4942 					  "Using MSIX interrupts with %d vectors\n", vectors);
4943 		scctx->isc_vectors = vectors;
4944 		scctx->isc_nrxqsets = rx_queues;
4945 		scctx->isc_ntxqsets = tx_queues;
4946 		scctx->isc_intr = IFLIB_INTR_MSIX;
4947 
4948 		return (vectors);
4949 	} else {
4950 		device_printf(dev, "failed to allocate %d msix vectors, err: %d - using MSI\n", vectors, err);
4951 	}
4952 msi:
4953 	vectors = pci_msi_count(dev);
4954 	scctx->isc_nrxqsets = 1;
4955 	scctx->isc_ntxqsets = 1;
4956 	scctx->isc_vectors = vectors;
4957 	if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0) {
4958 		device_printf(dev,"Using an MSI interrupt\n");
4959 		scctx->isc_intr = IFLIB_INTR_MSI;
4960 	} else {
4961 		device_printf(dev,"Using a Legacy interrupt\n");
4962 		scctx->isc_intr = IFLIB_INTR_LEGACY;
4963 	}
4964 
4965 	return (vectors);
4966 }
4967 
4968 char * ring_states[] = { "IDLE", "BUSY", "STALLED", "ABDICATED" };
4969 
4970 static int
4971 mp_ring_state_handler(SYSCTL_HANDLER_ARGS)
4972 {
4973 	int rc;
4974 	uint16_t *state = ((uint16_t *)oidp->oid_arg1);
4975 	struct sbuf *sb;
4976 	char *ring_state = "UNKNOWN";
4977 
4978 	/* XXX needed ? */
4979 	rc = sysctl_wire_old_buffer(req, 0);
4980 	MPASS(rc == 0);
4981 	if (rc != 0)
4982 		return (rc);
4983 	sb = sbuf_new_for_sysctl(NULL, NULL, 80, req);
4984 	MPASS(sb != NULL);
4985 	if (sb == NULL)
4986 		return (ENOMEM);
4987 	if (state[3] <= 3)
4988 		ring_state = ring_states[state[3]];
4989 
4990 	sbuf_printf(sb, "pidx_head: %04hd pidx_tail: %04hd cidx: %04hd state: %s",
4991 		    state[0], state[1], state[2], ring_state);
4992 	rc = sbuf_finish(sb);
4993 	sbuf_delete(sb);
4994         return(rc);
4995 }
4996 
4997 enum iflib_ndesc_handler {
4998 	IFLIB_NTXD_HANDLER,
4999 	IFLIB_NRXD_HANDLER,
5000 };
5001 
5002 static int
5003 mp_ndesc_handler(SYSCTL_HANDLER_ARGS)
5004 {
5005 	if_ctx_t ctx = (void *)arg1;
5006 	enum iflib_ndesc_handler type = arg2;
5007 	char buf[256] = {0};
5008 	uint16_t *ndesc;
5009 	char *p, *next;
5010 	int nqs, rc, i;
5011 
5012 	MPASS(type == IFLIB_NTXD_HANDLER || type == IFLIB_NRXD_HANDLER);
5013 
5014 	nqs = 8;
5015 	switch(type) {
5016 	case IFLIB_NTXD_HANDLER:
5017 		ndesc = ctx->ifc_sysctl_ntxds;
5018 		if (ctx->ifc_sctx)
5019 			nqs = ctx->ifc_sctx->isc_ntxqs;
5020 		break;
5021 	case IFLIB_NRXD_HANDLER:
5022 		ndesc = ctx->ifc_sysctl_nrxds;
5023 		if (ctx->ifc_sctx)
5024 			nqs = ctx->ifc_sctx->isc_nrxqs;
5025 		break;
5026 	}
5027 	if (nqs == 0)
5028 		nqs = 8;
5029 
5030 	for (i=0; i<8; i++) {
5031 		if (i >= nqs)
5032 			break;
5033 		if (i)
5034 			strcat(buf, ",");
5035 		sprintf(strchr(buf, 0), "%d", ndesc[i]);
5036 	}
5037 
5038 	rc = sysctl_handle_string(oidp, buf, sizeof(buf), req);
5039 	if (rc || req->newptr == NULL)
5040 		return rc;
5041 
5042 	for (i = 0, next = buf, p = strsep(&next, " ,"); i < 8 && p;
5043 	    i++, p = strsep(&next, " ,")) {
5044 		ndesc[i] = strtoul(p, NULL, 10);
5045 	}
5046 
5047 	return(rc);
5048 }
5049 
5050 #define NAME_BUFLEN 32
5051 static void
5052 iflib_add_device_sysctl_pre(if_ctx_t ctx)
5053 {
5054         device_t dev = iflib_get_dev(ctx);
5055 	struct sysctl_oid_list *child, *oid_list;
5056 	struct sysctl_ctx_list *ctx_list;
5057 	struct sysctl_oid *node;
5058 
5059 	ctx_list = device_get_sysctl_ctx(dev);
5060 	child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
5061 	ctx->ifc_sysctl_node = node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, "iflib",
5062 						      CTLFLAG_RD, NULL, "IFLIB fields");
5063 	oid_list = SYSCTL_CHILDREN(node);
5064 
5065 	SYSCTL_ADD_STRING(ctx_list, oid_list, OID_AUTO, "driver_version",
5066 		       CTLFLAG_RD, ctx->ifc_sctx->isc_driver_version, 0,
5067 		       "driver version");
5068 
5069 	SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_ntxqs",
5070 		       CTLFLAG_RWTUN, &ctx->ifc_sysctl_ntxqs, 0,
5071 			"# of txqs to use, 0 => use default #");
5072 	SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_nrxqs",
5073 		       CTLFLAG_RWTUN, &ctx->ifc_sysctl_nrxqs, 0,
5074 			"# of rxqs to use, 0 => use default #");
5075 	SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_qs_enable",
5076 		       CTLFLAG_RWTUN, &ctx->ifc_sysctl_qs_eq_override, 0,
5077                        "permit #txq != #rxq");
5078 
5079 	/* XXX change for per-queue sizes */
5080 	SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_ntxds",
5081 		       CTLTYPE_STRING|CTLFLAG_RWTUN, ctx, IFLIB_NTXD_HANDLER,
5082                        mp_ndesc_handler, "A",
5083                        "list of # of tx descriptors to use, 0 = use default #");
5084 	SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_nrxds",
5085 		       CTLTYPE_STRING|CTLFLAG_RWTUN, ctx, IFLIB_NRXD_HANDLER,
5086                        mp_ndesc_handler, "A",
5087                        "list of # of rx descriptors to use, 0 = use default #");
5088 }
5089 
5090 static void
5091 iflib_add_device_sysctl_post(if_ctx_t ctx)
5092 {
5093 	if_shared_ctx_t sctx = ctx->ifc_sctx;
5094 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
5095         device_t dev = iflib_get_dev(ctx);
5096 	struct sysctl_oid_list *child;
5097 	struct sysctl_ctx_list *ctx_list;
5098 	iflib_fl_t fl;
5099 	iflib_txq_t txq;
5100 	iflib_rxq_t rxq;
5101 	int i, j;
5102 	char namebuf[NAME_BUFLEN];
5103 	char *qfmt;
5104 	struct sysctl_oid *queue_node, *fl_node, *node;
5105 	struct sysctl_oid_list *queue_list, *fl_list;
5106 	ctx_list = device_get_sysctl_ctx(dev);
5107 
5108 	node = ctx->ifc_sysctl_node;
5109 	child = SYSCTL_CHILDREN(node);
5110 
5111 	if (scctx->isc_ntxqsets > 100)
5112 		qfmt = "txq%03d";
5113 	else if (scctx->isc_ntxqsets > 10)
5114 		qfmt = "txq%02d";
5115 	else
5116 		qfmt = "txq%d";
5117 	for (i = 0, txq = ctx->ifc_txqs; i < scctx->isc_ntxqsets; i++, txq++) {
5118 		snprintf(namebuf, NAME_BUFLEN, qfmt, i);
5119 		queue_node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, namebuf,
5120 					     CTLFLAG_RD, NULL, "Queue Name");
5121 		queue_list = SYSCTL_CHILDREN(queue_node);
5122 #if MEMORY_LOGGING
5123 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_dequeued",
5124 				CTLFLAG_RD,
5125 				&txq->ift_dequeued, "total mbufs freed");
5126 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_enqueued",
5127 				CTLFLAG_RD,
5128 				&txq->ift_enqueued, "total mbufs enqueued");
5129 #endif
5130 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "mbuf_defrag",
5131 				   CTLFLAG_RD,
5132 				   &txq->ift_mbuf_defrag, "# of times m_defrag was called");
5133 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "m_pullups",
5134 				   CTLFLAG_RD,
5135 				   &txq->ift_pullups, "# of times m_pullup was called");
5136 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "mbuf_defrag_failed",
5137 				   CTLFLAG_RD,
5138 				   &txq->ift_mbuf_defrag_failed, "# of times m_defrag failed");
5139 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "no_desc_avail",
5140 				   CTLFLAG_RD,
5141 				   &txq->ift_no_desc_avail, "# of times no descriptors were available");
5142 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "tx_map_failed",
5143 				   CTLFLAG_RD,
5144 				   &txq->ift_map_failed, "# of times dma map failed");
5145 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txd_encap_efbig",
5146 				   CTLFLAG_RD,
5147 				   &txq->ift_txd_encap_efbig, "# of times txd_encap returned EFBIG");
5148 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "no_tx_dma_setup",
5149 				   CTLFLAG_RD,
5150 				   &txq->ift_no_tx_dma_setup, "# of times map failed for other than EFBIG");
5151 		SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "txq_pidx",
5152 				   CTLFLAG_RD,
5153 				   &txq->ift_pidx, 1, "Producer Index");
5154 		SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "txq_cidx",
5155 				   CTLFLAG_RD,
5156 				   &txq->ift_cidx, 1, "Consumer Index");
5157 		SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "txq_cidx_processed",
5158 				   CTLFLAG_RD,
5159 				   &txq->ift_cidx_processed, 1, "Consumer Index seen by credit update");
5160 		SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "txq_in_use",
5161 				   CTLFLAG_RD,
5162 				   &txq->ift_in_use, 1, "descriptors in use");
5163 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_processed",
5164 				   CTLFLAG_RD,
5165 				   &txq->ift_processed, "descriptors procesed for clean");
5166 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_cleaned",
5167 				   CTLFLAG_RD,
5168 				   &txq->ift_cleaned, "total cleaned");
5169 		SYSCTL_ADD_PROC(ctx_list, queue_list, OID_AUTO, "ring_state",
5170 				CTLTYPE_STRING | CTLFLAG_RD, __DEVOLATILE(uint64_t *, &txq->ift_br[0]->state),
5171 				0, mp_ring_state_handler, "A", "soft ring state");
5172 		SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_enqueues",
5173 				       CTLFLAG_RD, &txq->ift_br[0]->enqueues,
5174 				       "# of enqueues to the mp_ring for this queue");
5175 		SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_drops",
5176 				       CTLFLAG_RD, &txq->ift_br[0]->drops,
5177 				       "# of drops in the mp_ring for this queue");
5178 		SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_starts",
5179 				       CTLFLAG_RD, &txq->ift_br[0]->starts,
5180 				       "# of normal consumer starts in the mp_ring for this queue");
5181 		SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_stalls",
5182 				       CTLFLAG_RD, &txq->ift_br[0]->stalls,
5183 					       "# of consumer stalls in the mp_ring for this queue");
5184 		SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_restarts",
5185 			       CTLFLAG_RD, &txq->ift_br[0]->restarts,
5186 				       "# of consumer restarts in the mp_ring for this queue");
5187 		SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_abdications",
5188 				       CTLFLAG_RD, &txq->ift_br[0]->abdications,
5189 				       "# of consumer abdications in the mp_ring for this queue");
5190 	}
5191 
5192 	if (scctx->isc_nrxqsets > 100)
5193 		qfmt = "rxq%03d";
5194 	else if (scctx->isc_nrxqsets > 10)
5195 		qfmt = "rxq%02d";
5196 	else
5197 		qfmt = "rxq%d";
5198 	for (i = 0, rxq = ctx->ifc_rxqs; i < scctx->isc_nrxqsets; i++, rxq++) {
5199 		snprintf(namebuf, NAME_BUFLEN, qfmt, i);
5200 		queue_node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, namebuf,
5201 					     CTLFLAG_RD, NULL, "Queue Name");
5202 		queue_list = SYSCTL_CHILDREN(queue_node);
5203 		if (sctx->isc_flags & IFLIB_HAS_RXCQ) {
5204 			SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "rxq_cq_pidx",
5205 				       CTLFLAG_RD,
5206 				       &rxq->ifr_cq_pidx, 1, "Producer Index");
5207 			SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "rxq_cq_cidx",
5208 				       CTLFLAG_RD,
5209 				       &rxq->ifr_cq_cidx, 1, "Consumer Index");
5210 		}
5211 
5212 		for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) {
5213 			snprintf(namebuf, NAME_BUFLEN, "rxq_fl%d", j);
5214 			fl_node = SYSCTL_ADD_NODE(ctx_list, queue_list, OID_AUTO, namebuf,
5215 						     CTLFLAG_RD, NULL, "freelist Name");
5216 			fl_list = SYSCTL_CHILDREN(fl_node);
5217 			SYSCTL_ADD_U16(ctx_list, fl_list, OID_AUTO, "pidx",
5218 				       CTLFLAG_RD,
5219 				       &fl->ifl_pidx, 1, "Producer Index");
5220 			SYSCTL_ADD_U16(ctx_list, fl_list, OID_AUTO, "cidx",
5221 				       CTLFLAG_RD,
5222 				       &fl->ifl_cidx, 1, "Consumer Index");
5223 			SYSCTL_ADD_U16(ctx_list, fl_list, OID_AUTO, "credits",
5224 				       CTLFLAG_RD,
5225 				       &fl->ifl_credits, 1, "credits available");
5226 #if MEMORY_LOGGING
5227 			SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_m_enqueued",
5228 					CTLFLAG_RD,
5229 					&fl->ifl_m_enqueued, "mbufs allocated");
5230 			SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_m_dequeued",
5231 					CTLFLAG_RD,
5232 					&fl->ifl_m_dequeued, "mbufs freed");
5233 			SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_cl_enqueued",
5234 					CTLFLAG_RD,
5235 					&fl->ifl_cl_enqueued, "clusters allocated");
5236 			SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_cl_dequeued",
5237 					CTLFLAG_RD,
5238 					&fl->ifl_cl_dequeued, "clusters freed");
5239 #endif
5240 
5241 		}
5242 	}
5243 
5244 }
5245