xref: /freebsd/sys/dev/ixl/if_ixl.c (revision 5f0216bd883edee71bf81051e3c20505e4820903)
1 /******************************************************************************
2 
3   Copyright (c) 2013-2015, Intel Corporation
4   All rights reserved.
5 
6   Redistribution and use in source and binary forms, with or without
7   modification, are permitted provided that the following conditions are met:
8 
9    1. Redistributions of source code must retain the above copyright notice,
10       this list of conditions and the following disclaimer.
11 
12    2. Redistributions in binary form must reproduce the above copyright
13       notice, this list of conditions and the following disclaimer in the
14       documentation and/or other materials provided with the distribution.
15 
16    3. Neither the name of the Intel Corporation nor the names of its
17       contributors may be used to endorse or promote products derived from
18       this software without specific prior written permission.
19 
20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30   POSSIBILITY OF SUCH DAMAGE.
31 
32 ******************************************************************************/
33 /*$FreeBSD$*/
34 
35 #ifndef IXL_STANDALONE_BUILD
36 #include "opt_inet.h"
37 #include "opt_inet6.h"
38 #include "opt_rss.h"
39 #endif
40 
41 #include "ixl.h"
42 #include "ixl_pf.h"
43 
44 #ifdef RSS
45 #include <net/rss_config.h>
46 #endif
47 
48 /*********************************************************************
49  *  Driver version
50  *********************************************************************/
51 char ixl_driver_version[] = "1.4.3";
52 
53 /*********************************************************************
54  *  PCI Device ID Table
55  *
56  *  Used by probe to select devices to load on
57  *  Last field stores an index into ixl_strings
58  *  Last entry must be all 0s
59  *
60  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
61  *********************************************************************/
62 
63 static ixl_vendor_info_t ixl_vendor_info_array[] =
64 {
65 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710, 0, 0, 0},
66 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_A, 0, 0, 0},
67 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B, 0, 0, 0},
68 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C, 0, 0, 0},
69 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A, 0, 0, 0},
70 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B, 0, 0, 0},
71 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C, 0, 0, 0},
72 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T, 0, 0, 0},
73 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T4, 0, 0, 0},
74 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_20G_KR2, 0, 0, 0},
75 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_20G_KR2_A, 0, 0, 0},
76 #ifdef X722_SUPPORT
77 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_X722, 0, 0, 0},
78 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_1G_BASE_T_X722, 0, 0, 0},
79 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T_X722, 0, 0, 0},
80 #endif
81 	/* required last entry */
82 	{0, 0, 0, 0, 0}
83 };
84 
85 /*********************************************************************
86  *  Table of branding strings
87  *********************************************************************/
88 
89 static char    *ixl_strings[] = {
90 	"Intel(R) Ethernet Connection XL710 Driver"
91 };
92 
93 
94 /*********************************************************************
95  *  Function prototypes
96  *********************************************************************/
97 static int      ixl_probe(device_t);
98 static int      ixl_attach(device_t);
99 static int      ixl_detach(device_t);
100 static int      ixl_shutdown(device_t);
101 static int	ixl_get_hw_capabilities(struct ixl_pf *);
102 static void	ixl_cap_txcsum_tso(struct ixl_vsi *, struct ifnet *, int);
103 static int      ixl_ioctl(struct ifnet *, u_long, caddr_t);
104 static void	ixl_init(void *);
105 static void	ixl_init_locked(struct ixl_pf *);
106 static void     ixl_stop(struct ixl_pf *);
107 static void     ixl_media_status(struct ifnet *, struct ifmediareq *);
108 static int      ixl_media_change(struct ifnet *);
109 static void     ixl_update_link_status(struct ixl_pf *);
110 static int      ixl_allocate_pci_resources(struct ixl_pf *);
111 static u16	ixl_get_bus_info(struct i40e_hw *, device_t);
112 static int	ixl_setup_stations(struct ixl_pf *);
113 static int	ixl_switch_config(struct ixl_pf *);
114 static int	ixl_initialize_vsi(struct ixl_vsi *);
115 static int	ixl_assign_vsi_msix(struct ixl_pf *);
116 static int	ixl_assign_vsi_legacy(struct ixl_pf *);
117 static int	ixl_init_msix(struct ixl_pf *);
118 static void	ixl_configure_msix(struct ixl_pf *);
119 static void	ixl_configure_itr(struct ixl_pf *);
120 static void	ixl_configure_legacy(struct ixl_pf *);
121 static void	ixl_free_pci_resources(struct ixl_pf *);
122 static void	ixl_local_timer(void *);
123 static int	ixl_setup_interface(device_t, struct ixl_vsi *);
124 static void	ixl_link_event(struct ixl_pf *, struct i40e_arq_event_info *);
125 static void	ixl_config_rss(struct ixl_vsi *);
126 static void	ixl_set_queue_rx_itr(struct ixl_queue *);
127 static void	ixl_set_queue_tx_itr(struct ixl_queue *);
128 static int	ixl_set_advertised_speeds(struct ixl_pf *, int);
129 
130 static int	ixl_enable_rings(struct ixl_vsi *);
131 static int	ixl_disable_rings(struct ixl_vsi *);
132 static void	ixl_enable_intr(struct ixl_vsi *);
133 static void	ixl_disable_intr(struct ixl_vsi *);
134 static void	ixl_disable_rings_intr(struct ixl_vsi *);
135 
136 static void     ixl_enable_adminq(struct i40e_hw *);
137 static void     ixl_disable_adminq(struct i40e_hw *);
138 static void     ixl_enable_queue(struct i40e_hw *, int);
139 static void     ixl_disable_queue(struct i40e_hw *, int);
140 static void     ixl_enable_legacy(struct i40e_hw *);
141 static void     ixl_disable_legacy(struct i40e_hw *);
142 
143 static void     ixl_set_promisc(struct ixl_vsi *);
144 static void     ixl_add_multi(struct ixl_vsi *);
145 static void     ixl_del_multi(struct ixl_vsi *);
146 static void	ixl_register_vlan(void *, struct ifnet *, u16);
147 static void	ixl_unregister_vlan(void *, struct ifnet *, u16);
148 static void	ixl_setup_vlan_filters(struct ixl_vsi *);
149 
150 static void	ixl_init_filters(struct ixl_vsi *);
151 static void	ixl_reconfigure_filters(struct ixl_vsi *vsi);
152 static void	ixl_add_filter(struct ixl_vsi *, u8 *, s16 vlan);
153 static void	ixl_del_filter(struct ixl_vsi *, u8 *, s16 vlan);
154 static void	ixl_add_hw_filters(struct ixl_vsi *, int, int);
155 static void	ixl_del_hw_filters(struct ixl_vsi *, int);
156 static struct ixl_mac_filter *
157 		ixl_find_filter(struct ixl_vsi *, u8 *, s16);
158 static void	ixl_add_mc_filter(struct ixl_vsi *, u8 *);
159 static void	ixl_free_mac_filters(struct ixl_vsi *vsi);
160 
161 
162 /* Sysctl debug interface */
163 static int	ixl_debug_info(SYSCTL_HANDLER_ARGS);
164 static void	ixl_print_debug_info(struct ixl_pf *);
165 
166 /* The MSI/X Interrupt handlers */
167 static void	ixl_intr(void *);
168 static void	ixl_msix_que(void *);
169 static void	ixl_msix_adminq(void *);
170 static void	ixl_handle_mdd_event(struct ixl_pf *);
171 
172 /* Deferred interrupt tasklets */
173 static void	ixl_do_adminq(void *, int);
174 
175 /* Sysctl handlers */
176 static int	ixl_set_flowcntl(SYSCTL_HANDLER_ARGS);
177 static int	ixl_set_advertise(SYSCTL_HANDLER_ARGS);
178 static int	ixl_current_speed(SYSCTL_HANDLER_ARGS);
179 static int	ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS);
180 
181 /* Statistics */
182 static void     ixl_add_hw_stats(struct ixl_pf *);
183 static void	ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *,
184 		    struct sysctl_oid_list *, struct i40e_hw_port_stats *);
185 static void	ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *,
186 		    struct sysctl_oid_list *,
187 		    struct i40e_eth_stats *);
188 static void	ixl_update_stats_counters(struct ixl_pf *);
189 static void	ixl_update_eth_stats(struct ixl_vsi *);
190 static void	ixl_update_vsi_stats(struct ixl_vsi *);
191 static void	ixl_pf_reset_stats(struct ixl_pf *);
192 static void	ixl_vsi_reset_stats(struct ixl_vsi *);
193 static void	ixl_stat_update48(struct i40e_hw *, u32, u32, bool,
194 		    u64 *, u64 *);
195 static void	ixl_stat_update32(struct i40e_hw *, u32, bool,
196 		    u64 *, u64 *);
197 
198 #ifdef IXL_DEBUG_SYSCTL
199 static int 	ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS);
200 static int	ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS);
201 static int	ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS);
202 static int	ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS);
203 static int	ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS);
204 #endif
205 
206 #ifdef PCI_IOV
207 static int	ixl_adminq_err_to_errno(enum i40e_admin_queue_err err);
208 
209 static int	ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t*);
210 static void	ixl_iov_uninit(device_t dev);
211 static int	ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t*);
212 
213 static void	ixl_handle_vf_msg(struct ixl_pf *,
214 		    struct i40e_arq_event_info *);
215 static void	ixl_handle_vflr(void *arg, int pending);
216 
217 static void	ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf);
218 static void	ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf);
219 #endif
220 
221 /*********************************************************************
222  *  FreeBSD Device Interface Entry Points
223  *********************************************************************/
224 
225 static device_method_t ixl_methods[] = {
226 	/* Device interface */
227 	DEVMETHOD(device_probe, ixl_probe),
228 	DEVMETHOD(device_attach, ixl_attach),
229 	DEVMETHOD(device_detach, ixl_detach),
230 	DEVMETHOD(device_shutdown, ixl_shutdown),
231 #ifdef PCI_IOV
232 	DEVMETHOD(pci_iov_init, ixl_iov_init),
233 	DEVMETHOD(pci_iov_uninit, ixl_iov_uninit),
234 	DEVMETHOD(pci_iov_add_vf, ixl_add_vf),
235 #endif
236 	{0, 0}
237 };
238 
239 static driver_t ixl_driver = {
240 	"ixl", ixl_methods, sizeof(struct ixl_pf),
241 };
242 
243 devclass_t ixl_devclass;
244 DRIVER_MODULE(ixl, pci, ixl_driver, ixl_devclass, 0, 0);
245 
246 MODULE_DEPEND(ixl, pci, 1, 1, 1);
247 MODULE_DEPEND(ixl, ether, 1, 1, 1);
248 #ifdef DEV_NETMAP
249 MODULE_DEPEND(ixl, netmap, 1, 1, 1);
250 #endif /* DEV_NETMAP */
251 
252 /*
253 ** Global reset mutex
254 */
255 static struct mtx ixl_reset_mtx;
256 
257 /*
258 ** TUNEABLE PARAMETERS:
259 */
260 
261 static SYSCTL_NODE(_hw, OID_AUTO, ixl, CTLFLAG_RD, 0,
262                    "IXL driver parameters");
263 
264 /*
265  * MSIX should be the default for best performance,
266  * but this allows it to be forced off for testing.
267  */
268 static int ixl_enable_msix = 1;
269 TUNABLE_INT("hw.ixl.enable_msix", &ixl_enable_msix);
270 SYSCTL_INT(_hw_ixl, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixl_enable_msix, 0,
271     "Enable MSI-X interrupts");
272 
273 /*
274 ** Number of descriptors per ring:
275 **   - TX and RX are the same size
276 */
277 static int ixl_ringsz = DEFAULT_RING;
278 TUNABLE_INT("hw.ixl.ringsz", &ixl_ringsz);
279 SYSCTL_INT(_hw_ixl, OID_AUTO, ring_size, CTLFLAG_RDTUN,
280     &ixl_ringsz, 0, "Descriptor Ring Size");
281 
282 /*
283 ** This can be set manually, if left as 0 the
284 ** number of queues will be calculated based
285 ** on cpus and msix vectors available.
286 */
287 int ixl_max_queues = 0;
288 TUNABLE_INT("hw.ixl.max_queues", &ixl_max_queues);
289 SYSCTL_INT(_hw_ixl, OID_AUTO, max_queues, CTLFLAG_RDTUN,
290     &ixl_max_queues, 0, "Number of Queues");
291 
292 /*
293 ** Controls for Interrupt Throttling
294 **	- true/false for dynamic adjustment
295 ** 	- default values for static ITR
296 */
297 int ixl_dynamic_rx_itr = 0;
298 TUNABLE_INT("hw.ixl.dynamic_rx_itr", &ixl_dynamic_rx_itr);
299 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN,
300     &ixl_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate");
301 
302 int ixl_dynamic_tx_itr = 0;
303 TUNABLE_INT("hw.ixl.dynamic_tx_itr", &ixl_dynamic_tx_itr);
304 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN,
305     &ixl_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate");
306 
307 int ixl_rx_itr = IXL_ITR_8K;
308 TUNABLE_INT("hw.ixl.rx_itr", &ixl_rx_itr);
309 SYSCTL_INT(_hw_ixl, OID_AUTO, rx_itr, CTLFLAG_RDTUN,
310     &ixl_rx_itr, 0, "RX Interrupt Rate");
311 
312 int ixl_tx_itr = IXL_ITR_4K;
313 TUNABLE_INT("hw.ixl.tx_itr", &ixl_tx_itr);
314 SYSCTL_INT(_hw_ixl, OID_AUTO, tx_itr, CTLFLAG_RDTUN,
315     &ixl_tx_itr, 0, "TX Interrupt Rate");
316 
317 #ifdef IXL_FDIR
318 static int ixl_enable_fdir = 1;
319 TUNABLE_INT("hw.ixl.enable_fdir", &ixl_enable_fdir);
320 /* Rate at which we sample */
321 int ixl_atr_rate = 20;
322 TUNABLE_INT("hw.ixl.atr_rate", &ixl_atr_rate);
323 #endif
324 
325 #ifdef DEV_NETMAP
326 #define NETMAP_IXL_MAIN /* only bring in one part of the netmap code */
327 #include <dev/netmap/if_ixl_netmap.h>
328 #endif /* DEV_NETMAP */
329 
330 static char *ixl_fc_string[6] = {
331 	"None",
332 	"Rx",
333 	"Tx",
334 	"Full",
335 	"Priority",
336 	"Default"
337 };
338 
339 static MALLOC_DEFINE(M_IXL, "ixl", "ixl driver allocations");
340 
341 static uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] =
342     {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
343 
344 /*********************************************************************
345  *  Device identification routine
346  *
347  *  ixl_probe determines if the driver should be loaded on
348  *  the hardware based on PCI vendor/device id of the device.
349  *
350  *  return BUS_PROBE_DEFAULT on success, positive on failure
351  *********************************************************************/
352 
353 static int
354 ixl_probe(device_t dev)
355 {
356 	ixl_vendor_info_t *ent;
357 
358 	u16	pci_vendor_id, pci_device_id;
359 	u16	pci_subvendor_id, pci_subdevice_id;
360 	char	device_name[256];
361 	static bool lock_init = FALSE;
362 
363 	INIT_DEBUGOUT("ixl_probe: begin");
364 
365 	pci_vendor_id = pci_get_vendor(dev);
366 	if (pci_vendor_id != I40E_INTEL_VENDOR_ID)
367 		return (ENXIO);
368 
369 	pci_device_id = pci_get_device(dev);
370 	pci_subvendor_id = pci_get_subvendor(dev);
371 	pci_subdevice_id = pci_get_subdevice(dev);
372 
373 	ent = ixl_vendor_info_array;
374 	while (ent->vendor_id != 0) {
375 		if ((pci_vendor_id == ent->vendor_id) &&
376 		    (pci_device_id == ent->device_id) &&
377 
378 		    ((pci_subvendor_id == ent->subvendor_id) ||
379 		     (ent->subvendor_id == 0)) &&
380 
381 		    ((pci_subdevice_id == ent->subdevice_id) ||
382 		     (ent->subdevice_id == 0))) {
383 			sprintf(device_name, "%s, Version - %s",
384 				ixl_strings[ent->index],
385 				ixl_driver_version);
386 			device_set_desc_copy(dev, device_name);
387 			/* One shot mutex init */
388 			if (lock_init == FALSE) {
389 				lock_init = TRUE;
390 				mtx_init(&ixl_reset_mtx,
391 				    "ixl_reset",
392 				    "IXL RESET Lock", MTX_DEF);
393 			}
394 			return (BUS_PROBE_DEFAULT);
395 		}
396 		ent++;
397 	}
398 	return (ENXIO);
399 }
400 
401 /*********************************************************************
402  *  Device initialization routine
403  *
404  *  The attach entry point is called when the driver is being loaded.
405  *  This routine identifies the type of hardware, allocates all resources
406  *  and initializes the hardware.
407  *
408  *  return 0 on success, positive on failure
409  *********************************************************************/
410 
411 static int
412 ixl_attach(device_t dev)
413 {
414 	struct ixl_pf	*pf;
415 	struct i40e_hw	*hw;
416 	struct ixl_vsi *vsi;
417 	u16		bus;
418 	int             error = 0;
419 #ifdef PCI_IOV
420 	nvlist_t	*pf_schema, *vf_schema;
421 	int		iov_error;
422 #endif
423 
424 	INIT_DEBUGOUT("ixl_attach: begin");
425 
426 	/* Allocate, clear, and link in our primary soft structure */
427 	pf = device_get_softc(dev);
428 	pf->dev = pf->osdep.dev = dev;
429 	hw = &pf->hw;
430 
431 	/*
432 	** Note this assumes we have a single embedded VSI,
433 	** this could be enhanced later to allocate multiple
434 	*/
435 	vsi = &pf->vsi;
436 	vsi->dev = pf->dev;
437 
438 	/* Core Lock Init*/
439 	IXL_PF_LOCK_INIT(pf, device_get_nameunit(dev));
440 
441 	/* Set up the timer callout */
442 	callout_init_mtx(&pf->timer, &pf->pf_mtx, 0);
443 
444 	/* Set up sysctls */
445 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
446 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
447 	    OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
448 	    pf, 0, ixl_set_flowcntl, "I", "Flow Control");
449 
450 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
451 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
452 	    OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
453 	    pf, 0, ixl_set_advertise, "I", "Advertised Speed");
454 
455 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
456 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
457 	    OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD,
458 	    pf, 0, ixl_current_speed, "A", "Current Port Speed");
459 
460 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
461 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
462 	    OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD,
463 	    pf, 0, ixl_sysctl_show_fw, "A", "Firmware version");
464 
465 	SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
466 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
467 	    OID_AUTO, "rx_itr", CTLFLAG_RW,
468 	    &ixl_rx_itr, IXL_ITR_8K, "RX ITR");
469 
470 	SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
471 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
472 	    OID_AUTO, "dynamic_rx_itr", CTLFLAG_RW,
473 	    &ixl_dynamic_rx_itr, 0, "Dynamic RX ITR");
474 
475 	SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
476 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
477 	    OID_AUTO, "tx_itr", CTLFLAG_RW,
478 	    &ixl_tx_itr, IXL_ITR_4K, "TX ITR");
479 
480 	SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
481 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
482 	    OID_AUTO, "dynamic_tx_itr", CTLFLAG_RW,
483 	    &ixl_dynamic_tx_itr, 0, "Dynamic TX ITR");
484 
485 #ifdef IXL_DEBUG_SYSCTL
486 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
487 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
488 	    OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, pf, 0,
489 	    ixl_debug_info, "I", "Debug Information");
490 
491 	/* Debug shared-code message level */
492 	SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
493 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
494 	    OID_AUTO, "debug_mask", CTLFLAG_RW,
495 	    &pf->hw.debug_mask, 0, "Debug Message Level");
496 
497 	SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
498 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
499 	    OID_AUTO, "vc_debug_level", CTLFLAG_RW, &pf->vc_debug_lvl,
500 	    0, "PF/VF Virtual Channel debug level");
501 
502 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
503 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
504 	    OID_AUTO, "link_status", CTLTYPE_STRING | CTLFLAG_RD,
505 	    pf, 0, ixl_sysctl_link_status, "A", "Current Link Status");
506 
507 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
508 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
509 	    OID_AUTO, "phy_abilities", CTLTYPE_STRING | CTLFLAG_RD,
510 	    pf, 0, ixl_sysctl_phy_abilities, "A", "PHY Abilities");
511 
512 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
513 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
514 	    OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD,
515 	    pf, 0, ixl_sysctl_sw_filter_list, "A", "SW Filter List");
516 
517 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
518 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
519 	    OID_AUTO, "hw_res_alloc", CTLTYPE_STRING | CTLFLAG_RD,
520 	    pf, 0, ixl_sysctl_hw_res_alloc, "A", "HW Resource Allocation");
521 
522 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
523 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
524 	    OID_AUTO, "switch_config", CTLTYPE_STRING | CTLFLAG_RD,
525 	    pf, 0, ixl_sysctl_switch_config, "A", "HW Switch Configuration");
526 #endif
527 
528 	/* Save off the PCI information */
529 	hw->vendor_id = pci_get_vendor(dev);
530 	hw->device_id = pci_get_device(dev);
531 	hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
532 	hw->subsystem_vendor_id =
533 	    pci_read_config(dev, PCIR_SUBVEND_0, 2);
534 	hw->subsystem_device_id =
535 	    pci_read_config(dev, PCIR_SUBDEV_0, 2);
536 
537 	hw->bus.device = pci_get_slot(dev);
538 	hw->bus.func = pci_get_function(dev);
539 
540 	pf->vc_debug_lvl = 1;
541 
542 	/* Do PCI setup - map BAR0, etc */
543 	if (ixl_allocate_pci_resources(pf)) {
544 		device_printf(dev, "Allocation of PCI resources failed\n");
545 		error = ENXIO;
546 		goto err_out;
547 	}
548 
549 	/* Establish a clean starting point */
550 	i40e_clear_hw(hw);
551 	error = i40e_pf_reset(hw);
552 	if (error) {
553 		device_printf(dev,"PF reset failure %x\n", error);
554 		error = EIO;
555 		goto err_out;
556 	}
557 
558 	/* Set admin queue parameters */
559 	hw->aq.num_arq_entries = IXL_AQ_LEN;
560 	hw->aq.num_asq_entries = IXL_AQ_LEN;
561 	hw->aq.arq_buf_size = IXL_AQ_BUFSZ;
562 	hw->aq.asq_buf_size = IXL_AQ_BUFSZ;
563 
564 	/* Initialize the shared code */
565 	error = i40e_init_shared_code(hw);
566 	if (error) {
567 		device_printf(dev,"Unable to initialize the shared code\n");
568 		error = EIO;
569 		goto err_out;
570 	}
571 
572 	/* Set up the admin queue */
573 	error = i40e_init_adminq(hw);
574 	if (error) {
575 		device_printf(dev, "The driver for the device stopped "
576 		    "because the NVM image is newer than expected.\n"
577 		    "You must install the most recent version of "
578 		    " the network driver.\n");
579 		goto err_out;
580 	}
581 	device_printf(dev, "%s\n", ixl_fw_version_str(hw));
582 
583         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
584 	    hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR)
585 		device_printf(dev, "The driver for the device detected "
586 		    "a newer version of the NVM image than expected.\n"
587 		    "Please install the most recent version of the network driver.\n");
588 	else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR ||
589 	    hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1))
590 		device_printf(dev, "The driver for the device detected "
591 		    "an older version of the NVM image than expected.\n"
592 		    "Please update the NVM image.\n");
593 
594 	/* Clear PXE mode */
595 	i40e_clear_pxe_mode(hw);
596 
597 	/* Get capabilities from the device */
598 	error = ixl_get_hw_capabilities(pf);
599 	if (error) {
600 		device_printf(dev, "HW capabilities failure!\n");
601 		goto err_get_cap;
602 	}
603 
604 	/* Set up host memory cache */
605 	error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
606 	    hw->func_caps.num_rx_qp, 0, 0);
607 	if (error) {
608 		device_printf(dev, "init_lan_hmc failed: %d\n", error);
609 		goto err_get_cap;
610 	}
611 
612 	error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
613 	if (error) {
614 		device_printf(dev, "configure_lan_hmc failed: %d\n", error);
615 		goto err_mac_hmc;
616 	}
617 
618 	/* Disable LLDP from the firmware */
619 	i40e_aq_stop_lldp(hw, TRUE, NULL);
620 
621 	i40e_get_mac_addr(hw, hw->mac.addr);
622 	error = i40e_validate_mac_addr(hw->mac.addr);
623 	if (error) {
624 		device_printf(dev, "validate_mac_addr failed: %d\n", error);
625 		goto err_mac_hmc;
626 	}
627 	bcopy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN);
628 	i40e_get_port_mac_addr(hw, hw->mac.port_addr);
629 
630 	/* Set up VSI and queues */
631 	if (ixl_setup_stations(pf) != 0) {
632 		device_printf(dev, "setup stations failed!\n");
633 		error = ENOMEM;
634 		goto err_mac_hmc;
635 	}
636 
637 	/* Initialize mac filter list for VSI */
638 	SLIST_INIT(&vsi->ftl);
639 
640 	/* Set up interrupt routing here */
641 	if (pf->msix > 1)
642 		error = ixl_assign_vsi_msix(pf);
643 	else
644 		error = ixl_assign_vsi_legacy(pf);
645 	if (error)
646 		goto err_late;
647 
648 	if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) ||
649 	    (hw->aq.fw_maj_ver < 4)) {
650 		i40e_msec_delay(75);
651 		error = i40e_aq_set_link_restart_an(hw, TRUE, NULL);
652 		if (error)
653 			device_printf(dev, "link restart failed, aq_err=%d\n",
654 			    pf->hw.aq.asq_last_status);
655 	}
656 
657 	/* Determine link state */
658 	i40e_aq_get_link_info(hw, TRUE, NULL, NULL);
659 	i40e_get_link_status(hw, &pf->link_up);
660 
661 	/* Setup OS specific network interface */
662 	if (ixl_setup_interface(dev, vsi) != 0) {
663 		device_printf(dev, "interface setup failed!\n");
664 		error = EIO;
665 		goto err_late;
666 	}
667 
668 	error = ixl_switch_config(pf);
669 	if (error) {
670 		device_printf(dev, "Initial switch config failed: %d\n", error);
671 		goto err_mac_hmc;
672 	}
673 
674 	/* Limit phy interrupts to link and modules failure */
675 	error = i40e_aq_set_phy_int_mask(hw,
676 	    I40E_AQ_EVENT_LINK_UPDOWN | I40E_AQ_EVENT_MODULE_QUAL_FAIL, NULL);
677         if (error)
678 		device_printf(dev, "set phy mask failed: %d\n", error);
679 
680 	/* Get the bus configuration and set the shared code */
681 	bus = ixl_get_bus_info(hw, dev);
682 	i40e_set_pci_config_data(hw, bus);
683 
684 	/* Initialize statistics */
685 	ixl_pf_reset_stats(pf);
686 	ixl_update_stats_counters(pf);
687 	ixl_add_hw_stats(pf);
688 
689 	/* Register for VLAN events */
690 	vsi->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
691 	    ixl_register_vlan, vsi, EVENTHANDLER_PRI_FIRST);
692 	vsi->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
693 	    ixl_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST);
694 
695 #ifdef PCI_IOV
696 	/* SR-IOV is only supported when MSI-X is in use. */
697 	if (pf->msix > 1) {
698 		pf_schema = pci_iov_schema_alloc_node();
699 		vf_schema = pci_iov_schema_alloc_node();
700 		pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL);
701 		pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof",
702 		    IOV_SCHEMA_HASDEFAULT, TRUE);
703 		pci_iov_schema_add_bool(vf_schema, "allow-set-mac",
704 		    IOV_SCHEMA_HASDEFAULT, FALSE);
705 		pci_iov_schema_add_bool(vf_schema, "allow-promisc",
706 		    IOV_SCHEMA_HASDEFAULT, FALSE);
707 
708 		iov_error = pci_iov_attach(dev, pf_schema, vf_schema);
709 		if (iov_error != 0)
710 			device_printf(dev,
711 			    "Failed to initialize SR-IOV (error=%d)\n",
712 			    iov_error);
713 	}
714 #endif
715 
716 #ifdef DEV_NETMAP
717 	ixl_netmap_attach(vsi);
718 #endif /* DEV_NETMAP */
719 	INIT_DEBUGOUT("ixl_attach: end");
720 	return (0);
721 
722 err_late:
723 	if (vsi->ifp != NULL)
724 		if_free(vsi->ifp);
725 err_mac_hmc:
726 	i40e_shutdown_lan_hmc(hw);
727 err_get_cap:
728 	i40e_shutdown_adminq(hw);
729 err_out:
730 	ixl_free_pci_resources(pf);
731 	ixl_free_vsi(vsi);
732 	IXL_PF_LOCK_DESTROY(pf);
733 	return (error);
734 }
735 
736 /*********************************************************************
737  *  Device removal routine
738  *
739  *  The detach entry point is called when the driver is being removed.
740  *  This routine stops the adapter and deallocates all the resources
741  *  that were allocated for driver operation.
742  *
743  *  return 0 on success, positive on failure
744  *********************************************************************/
745 
746 static int
747 ixl_detach(device_t dev)
748 {
749 	struct ixl_pf		*pf = device_get_softc(dev);
750 	struct i40e_hw		*hw = &pf->hw;
751 	struct ixl_vsi		*vsi = &pf->vsi;
752 	struct ixl_queue	*que = vsi->queues;
753 	i40e_status		status;
754 #ifdef PCI_IOV
755 	int			error;
756 #endif
757 
758 	INIT_DEBUGOUT("ixl_detach: begin");
759 
760 	/* Make sure VLANS are not using driver */
761 	if (vsi->ifp->if_vlantrunk != NULL) {
762 		device_printf(dev,"Vlan in use, detach first\n");
763 		return (EBUSY);
764 	}
765 
766 #ifdef PCI_IOV
767 	error = pci_iov_detach(dev);
768 	if (error != 0) {
769 		device_printf(dev, "SR-IOV in use; detach first.\n");
770 		return (error);
771 	}
772 #endif
773 
774 	ether_ifdetach(vsi->ifp);
775 	if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING) {
776 		IXL_PF_LOCK(pf);
777 		ixl_stop(pf);
778 		IXL_PF_UNLOCK(pf);
779 	}
780 
781 	for (int i = 0; i < vsi->num_queues; i++, que++) {
782 		if (que->tq) {
783 			taskqueue_drain(que->tq, &que->task);
784 			taskqueue_drain(que->tq, &que->tx_task);
785 			taskqueue_free(que->tq);
786 		}
787 	}
788 
789 	/* Shutdown LAN HMC */
790 	status = i40e_shutdown_lan_hmc(hw);
791 	if (status)
792 		device_printf(dev,
793 		    "Shutdown LAN HMC failed with code %d\n", status);
794 
795 	/* Shutdown admin queue */
796 	status = i40e_shutdown_adminq(hw);
797 	if (status)
798 		device_printf(dev,
799 		    "Shutdown Admin queue failed with code %d\n", status);
800 
801 	/* Unregister VLAN events */
802 	if (vsi->vlan_attach != NULL)
803 		EVENTHANDLER_DEREGISTER(vlan_config, vsi->vlan_attach);
804 	if (vsi->vlan_detach != NULL)
805 		EVENTHANDLER_DEREGISTER(vlan_unconfig, vsi->vlan_detach);
806 
807 	callout_drain(&pf->timer);
808 #ifdef DEV_NETMAP
809 	netmap_detach(vsi->ifp);
810 #endif /* DEV_NETMAP */
811 	ixl_free_pci_resources(pf);
812 	bus_generic_detach(dev);
813 	if_free(vsi->ifp);
814 	ixl_free_vsi(vsi);
815 	IXL_PF_LOCK_DESTROY(pf);
816 	return (0);
817 }
818 
819 /*********************************************************************
820  *
821  *  Shutdown entry point
822  *
823  **********************************************************************/
824 
825 static int
826 ixl_shutdown(device_t dev)
827 {
828 	struct ixl_pf *pf = device_get_softc(dev);
829 	IXL_PF_LOCK(pf);
830 	ixl_stop(pf);
831 	IXL_PF_UNLOCK(pf);
832 	return (0);
833 }
834 
835 
836 /*********************************************************************
837  *
838  *  Get the hardware capabilities
839  *
840  **********************************************************************/
841 
842 static int
843 ixl_get_hw_capabilities(struct ixl_pf *pf)
844 {
845 	struct i40e_aqc_list_capabilities_element_resp *buf;
846 	struct i40e_hw	*hw = &pf->hw;
847 	device_t 	dev = pf->dev;
848 	int             error, len;
849 	u16		needed;
850 	bool		again = TRUE;
851 
852 	len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
853 retry:
854 	if (!(buf = (struct i40e_aqc_list_capabilities_element_resp *)
855 	    malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO))) {
856 		device_printf(dev, "Unable to allocate cap memory\n");
857                 return (ENOMEM);
858 	}
859 
860 	/* This populates the hw struct */
861         error = i40e_aq_discover_capabilities(hw, buf, len,
862 	    &needed, i40e_aqc_opc_list_func_capabilities, NULL);
863 	free(buf, M_DEVBUF);
864 	if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) &&
865 	    (again == TRUE)) {
866 		/* retry once with a larger buffer */
867 		again = FALSE;
868 		len = needed;
869 		goto retry;
870 	} else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
871 		device_printf(dev, "capability discovery failed: %d\n",
872 		    pf->hw.aq.asq_last_status);
873 		return (ENODEV);
874 	}
875 
876 	/* Capture this PF's starting queue pair */
877 	pf->qbase = hw->func_caps.base_queue;
878 
879 #ifdef IXL_DEBUG
880 	device_printf(dev,"pf_id=%d, num_vfs=%d, msix_pf=%d, "
881 	    "msix_vf=%d, fd_g=%d, fd_b=%d, tx_qp=%d rx_qp=%d qbase=%d\n",
882 	    hw->pf_id, hw->func_caps.num_vfs,
883 	    hw->func_caps.num_msix_vectors,
884 	    hw->func_caps.num_msix_vectors_vf,
885 	    hw->func_caps.fd_filters_guaranteed,
886 	    hw->func_caps.fd_filters_best_effort,
887 	    hw->func_caps.num_tx_qp,
888 	    hw->func_caps.num_rx_qp,
889 	    hw->func_caps.base_queue);
890 #endif
891 	return (error);
892 }
893 
894 static void
895 ixl_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask)
896 {
897 	device_t 	dev = vsi->dev;
898 
899 	/* Enable/disable TXCSUM/TSO4 */
900 	if (!(ifp->if_capenable & IFCAP_TXCSUM)
901 	    && !(ifp->if_capenable & IFCAP_TSO4)) {
902 		if (mask & IFCAP_TXCSUM) {
903 			ifp->if_capenable |= IFCAP_TXCSUM;
904 			/* enable TXCSUM, restore TSO if previously enabled */
905 			if (vsi->flags & IXL_FLAGS_KEEP_TSO4) {
906 				vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
907 				ifp->if_capenable |= IFCAP_TSO4;
908 			}
909 		}
910 		else if (mask & IFCAP_TSO4) {
911 			ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4);
912 			vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
913 			device_printf(dev,
914 			    "TSO4 requires txcsum, enabling both...\n");
915 		}
916 	} else if((ifp->if_capenable & IFCAP_TXCSUM)
917 	    && !(ifp->if_capenable & IFCAP_TSO4)) {
918 		if (mask & IFCAP_TXCSUM)
919 			ifp->if_capenable &= ~IFCAP_TXCSUM;
920 		else if (mask & IFCAP_TSO4)
921 			ifp->if_capenable |= IFCAP_TSO4;
922 	} else if((ifp->if_capenable & IFCAP_TXCSUM)
923 	    && (ifp->if_capenable & IFCAP_TSO4)) {
924 		if (mask & IFCAP_TXCSUM) {
925 			vsi->flags |= IXL_FLAGS_KEEP_TSO4;
926 			ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4);
927 			device_printf(dev,
928 			    "TSO4 requires txcsum, disabling both...\n");
929 		} else if (mask & IFCAP_TSO4)
930 			ifp->if_capenable &= ~IFCAP_TSO4;
931 	}
932 
933 	/* Enable/disable TXCSUM_IPV6/TSO6 */
934 	if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6)
935 	    && !(ifp->if_capenable & IFCAP_TSO6)) {
936 		if (mask & IFCAP_TXCSUM_IPV6) {
937 			ifp->if_capenable |= IFCAP_TXCSUM_IPV6;
938 			if (vsi->flags & IXL_FLAGS_KEEP_TSO6) {
939 				vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
940 				ifp->if_capenable |= IFCAP_TSO6;
941 			}
942 		} else if (mask & IFCAP_TSO6) {
943 			ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
944 			vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
945 			device_printf(dev,
946 			    "TSO6 requires txcsum6, enabling both...\n");
947 		}
948 	} else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
949 	    && !(ifp->if_capenable & IFCAP_TSO6)) {
950 		if (mask & IFCAP_TXCSUM_IPV6)
951 			ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6;
952 		else if (mask & IFCAP_TSO6)
953 			ifp->if_capenable |= IFCAP_TSO6;
954 	} else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
955 	    && (ifp->if_capenable & IFCAP_TSO6)) {
956 		if (mask & IFCAP_TXCSUM_IPV6) {
957 			vsi->flags |= IXL_FLAGS_KEEP_TSO6;
958 			ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
959 			device_printf(dev,
960 			    "TSO6 requires txcsum6, disabling both...\n");
961 		} else if (mask & IFCAP_TSO6)
962 			ifp->if_capenable &= ~IFCAP_TSO6;
963 	}
964 }
965 
966 /*********************************************************************
967  *  Ioctl entry point
968  *
969  *  ixl_ioctl is called when the user wants to configure the
970  *  interface.
971  *
972  *  return 0 on success, positive on failure
973  **********************************************************************/
974 
975 static int
976 ixl_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
977 {
978 	struct ixl_vsi	*vsi = ifp->if_softc;
979 	struct ixl_pf	*pf = vsi->back;
980 	struct ifreq	*ifr = (struct ifreq *) data;
981 #if defined(INET) || defined(INET6)
982 	struct ifaddr *ifa = (struct ifaddr *)data;
983 	bool		avoid_reset = FALSE;
984 #endif
985 	int             error = 0;
986 
987 	switch (command) {
988 
989         case SIOCSIFADDR:
990 #ifdef INET
991 		if (ifa->ifa_addr->sa_family == AF_INET)
992 			avoid_reset = TRUE;
993 #endif
994 #ifdef INET6
995 		if (ifa->ifa_addr->sa_family == AF_INET6)
996 			avoid_reset = TRUE;
997 #endif
998 #if defined(INET) || defined(INET6)
999 		/*
1000 		** Calling init results in link renegotiation,
1001 		** so we avoid doing it when possible.
1002 		*/
1003 		if (avoid_reset) {
1004 			ifp->if_flags |= IFF_UP;
1005 			if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1006 				ixl_init(pf);
1007 #ifdef INET
1008 			if (!(ifp->if_flags & IFF_NOARP))
1009 				arp_ifinit(ifp, ifa);
1010 #endif
1011 		} else
1012 			error = ether_ioctl(ifp, command, data);
1013 		break;
1014 #endif
1015 	case SIOCSIFMTU:
1016 		IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
1017 		if (ifr->ifr_mtu > IXL_MAX_FRAME -
1018 		   ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) {
1019 			error = EINVAL;
1020 		} else {
1021 			IXL_PF_LOCK(pf);
1022 			ifp->if_mtu = ifr->ifr_mtu;
1023 			vsi->max_frame_size =
1024 				ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
1025 			    + ETHER_VLAN_ENCAP_LEN;
1026 			ixl_init_locked(pf);
1027 			IXL_PF_UNLOCK(pf);
1028 		}
1029 		break;
1030 	case SIOCSIFFLAGS:
1031 		IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
1032 		IXL_PF_LOCK(pf);
1033 		if (ifp->if_flags & IFF_UP) {
1034 			if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1035 				if ((ifp->if_flags ^ pf->if_flags) &
1036 				    (IFF_PROMISC | IFF_ALLMULTI)) {
1037 					ixl_set_promisc(vsi);
1038 				}
1039 			} else
1040 				ixl_init_locked(pf);
1041 		} else
1042 			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1043 				ixl_stop(pf);
1044 		pf->if_flags = ifp->if_flags;
1045 		IXL_PF_UNLOCK(pf);
1046 		break;
1047 	case SIOCADDMULTI:
1048 		IOCTL_DEBUGOUT("ioctl: SIOCADDMULTI");
1049 		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1050 			IXL_PF_LOCK(pf);
1051 			ixl_disable_intr(vsi);
1052 			ixl_add_multi(vsi);
1053 			ixl_enable_intr(vsi);
1054 			IXL_PF_UNLOCK(pf);
1055 		}
1056 		break;
1057 	case SIOCDELMULTI:
1058 		IOCTL_DEBUGOUT("ioctl: SIOCDELMULTI");
1059 		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1060 			IXL_PF_LOCK(pf);
1061 			ixl_disable_intr(vsi);
1062 			ixl_del_multi(vsi);
1063 			ixl_enable_intr(vsi);
1064 			IXL_PF_UNLOCK(pf);
1065 		}
1066 		break;
1067 	case SIOCSIFMEDIA:
1068 	case SIOCGIFMEDIA:
1069 #ifdef IFM_ETH_XTYPE
1070 	case SIOCGIFXMEDIA:
1071 #endif
1072 		IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1073 		error = ifmedia_ioctl(ifp, ifr, &vsi->media, command);
1074 		break;
1075 	case SIOCSIFCAP:
1076 	{
1077 		int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1078 		IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1079 
1080 		ixl_cap_txcsum_tso(vsi, ifp, mask);
1081 
1082 		if (mask & IFCAP_RXCSUM)
1083 			ifp->if_capenable ^= IFCAP_RXCSUM;
1084 		if (mask & IFCAP_RXCSUM_IPV6)
1085 			ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1086 		if (mask & IFCAP_LRO)
1087 			ifp->if_capenable ^= IFCAP_LRO;
1088 		if (mask & IFCAP_VLAN_HWTAGGING)
1089 			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1090 		if (mask & IFCAP_VLAN_HWFILTER)
1091 			ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1092 		if (mask & IFCAP_VLAN_HWTSO)
1093 			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1094 		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1095 			IXL_PF_LOCK(pf);
1096 			ixl_init_locked(pf);
1097 			IXL_PF_UNLOCK(pf);
1098 		}
1099 		VLAN_CAPABILITIES(ifp);
1100 
1101 		break;
1102 	}
1103 
1104 	default:
1105 		IOCTL_DEBUGOUT("ioctl: UNKNOWN (0x%X)\n", (int)command);
1106 		error = ether_ioctl(ifp, command, data);
1107 		break;
1108 	}
1109 
1110 	return (error);
1111 }
1112 
1113 
1114 /*********************************************************************
1115  *  Init entry point
1116  *
1117  *  This routine is used in two ways. It is used by the stack as
1118  *  init entry point in network interface structure. It is also used
1119  *  by the driver as a hw/sw initialization routine to get to a
1120  *  consistent state.
1121  *
1122  *  return 0 on success, positive on failure
1123  **********************************************************************/
1124 
1125 static void
1126 ixl_init_locked(struct ixl_pf *pf)
1127 {
1128 	struct i40e_hw	*hw = &pf->hw;
1129 	struct ixl_vsi	*vsi = &pf->vsi;
1130 	struct ifnet	*ifp = vsi->ifp;
1131 	device_t 	dev = pf->dev;
1132 	struct i40e_filter_control_settings	filter;
1133 	u8		tmpaddr[ETHER_ADDR_LEN];
1134 	int		ret;
1135 
1136 	mtx_assert(&pf->pf_mtx, MA_OWNED);
1137 	INIT_DEBUGOUT("ixl_init: begin");
1138 	ixl_stop(pf);
1139 
1140 	/* Get the latest mac address... User might use a LAA */
1141 	bcopy(IF_LLADDR(vsi->ifp), tmpaddr,
1142 	      I40E_ETH_LENGTH_OF_ADDRESS);
1143 	if (!cmp_etheraddr(hw->mac.addr, tmpaddr) &&
1144 	    i40e_validate_mac_addr(tmpaddr)) {
1145 		bcopy(tmpaddr, hw->mac.addr,
1146 		    I40E_ETH_LENGTH_OF_ADDRESS);
1147 		ret = i40e_aq_mac_address_write(hw,
1148 		    I40E_AQC_WRITE_TYPE_LAA_ONLY,
1149 		    hw->mac.addr, NULL);
1150 		if (ret) {
1151 			device_printf(dev, "LLA address"
1152 			 "change failed!!\n");
1153 			return;
1154 		}
1155 	}
1156 
1157 	/* Set the various hardware offload abilities */
1158 	ifp->if_hwassist = 0;
1159 	if (ifp->if_capenable & IFCAP_TSO)
1160 		ifp->if_hwassist |= CSUM_TSO;
1161 	if (ifp->if_capenable & IFCAP_TXCSUM)
1162 		ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1163 	if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
1164 		ifp->if_hwassist |= (CSUM_TCP_IPV6 | CSUM_UDP_IPV6);
1165 
1166 	/* Set up the device filtering */
1167 	bzero(&filter, sizeof(filter));
1168 	filter.enable_ethtype = TRUE;
1169 	filter.enable_macvlan = TRUE;
1170 #ifdef IXL_FDIR
1171 	filter.enable_fdir = TRUE;
1172 #endif
1173 	if (i40e_set_filter_control(hw, &filter))
1174 		device_printf(dev, "set_filter_control() failed\n");
1175 
1176 	/* Set up RSS */
1177 	ixl_config_rss(vsi);
1178 
1179 	/*
1180 	** Prepare the VSI: rings, hmc contexts, etc...
1181 	*/
1182 	if (ixl_initialize_vsi(vsi)) {
1183 		device_printf(dev, "initialize vsi failed!!\n");
1184 		return;
1185 	}
1186 
1187 	/* Add protocol filters to list */
1188 	ixl_init_filters(vsi);
1189 
1190 	/* Setup vlan's if needed */
1191 	ixl_setup_vlan_filters(vsi);
1192 
1193 	/* Start the local timer */
1194 	callout_reset(&pf->timer, hz, ixl_local_timer, pf);
1195 
1196 	/* Set up MSI/X routing and the ITR settings */
1197 	if (ixl_enable_msix) {
1198 		ixl_configure_msix(pf);
1199 		ixl_configure_itr(pf);
1200 	} else
1201 		ixl_configure_legacy(pf);
1202 
1203 	ixl_enable_rings(vsi);
1204 
1205 	i40e_aq_set_default_vsi(hw, vsi->seid, NULL);
1206 
1207 	ixl_reconfigure_filters(vsi);
1208 
1209 	/* Set MTU in hardware*/
1210 	int aq_error = i40e_aq_set_mac_config(hw, vsi->max_frame_size,
1211 	    TRUE, 0, NULL);
1212 	if (aq_error)
1213 		device_printf(vsi->dev,
1214 			"aq_set_mac_config in init error, code %d\n",
1215 		    aq_error);
1216 
1217 	/* And now turn on interrupts */
1218 	ixl_enable_intr(vsi);
1219 
1220 	/* Now inform the stack we're ready */
1221 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
1222 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1223 
1224 	return;
1225 }
1226 
1227 static void
1228 ixl_init(void *arg)
1229 {
1230 	struct ixl_pf *pf = arg;
1231 
1232 	IXL_PF_LOCK(pf);
1233 	ixl_init_locked(pf);
1234 	IXL_PF_UNLOCK(pf);
1235 	return;
1236 }
1237 
1238 /*
1239 **
1240 ** MSIX Interrupt Handlers and Tasklets
1241 **
1242 */
1243 static void
1244 ixl_handle_que(void *context, int pending)
1245 {
1246 	struct ixl_queue *que = context;
1247 	struct ixl_vsi *vsi = que->vsi;
1248 	struct i40e_hw  *hw = vsi->hw;
1249 	struct tx_ring  *txr = &que->txr;
1250 	struct ifnet    *ifp = vsi->ifp;
1251 	bool		more;
1252 
1253 	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1254 		more = ixl_rxeof(que, IXL_RX_LIMIT);
1255 		IXL_TX_LOCK(txr);
1256 		ixl_txeof(que);
1257 		if (!drbr_empty(ifp, txr->br))
1258 			ixl_mq_start_locked(ifp, txr);
1259 		IXL_TX_UNLOCK(txr);
1260 		if (more) {
1261 			taskqueue_enqueue(que->tq, &que->task);
1262 			return;
1263 		}
1264 	}
1265 
1266 	/* Reenable this interrupt - hmmm */
1267 	ixl_enable_queue(hw, que->me);
1268 	return;
1269 }
1270 
1271 
1272 /*********************************************************************
1273  *
1274  *  Legacy Interrupt Service routine
1275  *
1276  **********************************************************************/
1277 void
1278 ixl_intr(void *arg)
1279 {
1280 	struct ixl_pf		*pf = arg;
1281 	struct i40e_hw		*hw =  &pf->hw;
1282 	struct ixl_vsi		*vsi = &pf->vsi;
1283 	struct ixl_queue	*que = vsi->queues;
1284 	struct ifnet		*ifp = vsi->ifp;
1285 	struct tx_ring		*txr = &que->txr;
1286         u32			reg, icr0, mask;
1287 	bool			more_tx, more_rx;
1288 
1289 	++que->irqs;
1290 
1291 	/* Protect against spurious interrupts */
1292 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1293 		return;
1294 
1295 	icr0 = rd32(hw, I40E_PFINT_ICR0);
1296 
1297 	reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1298 	reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1299 	wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1300 
1301         mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1302 
1303 #ifdef PCI_IOV
1304 	if (icr0 & I40E_PFINT_ICR0_VFLR_MASK)
1305 		taskqueue_enqueue(pf->tq, &pf->vflr_task);
1306 #endif
1307 
1308 	if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
1309 		taskqueue_enqueue(pf->tq, &pf->adminq);
1310 		return;
1311 	}
1312 
1313 	more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1314 
1315 	IXL_TX_LOCK(txr);
1316 	more_tx = ixl_txeof(que);
1317 	if (!drbr_empty(vsi->ifp, txr->br))
1318 		more_tx = 1;
1319 	IXL_TX_UNLOCK(txr);
1320 
1321 	/* re-enable other interrupt causes */
1322 	wr32(hw, I40E_PFINT_ICR0_ENA, mask);
1323 
1324 	/* And now the queues */
1325 	reg = rd32(hw, I40E_QINT_RQCTL(0));
1326 	reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1327 	wr32(hw, I40E_QINT_RQCTL(0), reg);
1328 
1329 	reg = rd32(hw, I40E_QINT_TQCTL(0));
1330 	reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1331 	reg &= ~I40E_PFINT_ICR0_INTEVENT_MASK;
1332 	wr32(hw, I40E_QINT_TQCTL(0), reg);
1333 
1334 	ixl_enable_legacy(hw);
1335 
1336 	return;
1337 }
1338 
1339 
1340 /*********************************************************************
1341  *
1342  *  MSIX VSI Interrupt Service routine
1343  *
1344  **********************************************************************/
1345 void
1346 ixl_msix_que(void *arg)
1347 {
1348 	struct ixl_queue	*que = arg;
1349 	struct ixl_vsi	*vsi = que->vsi;
1350 	struct i40e_hw	*hw = vsi->hw;
1351 	struct tx_ring	*txr = &que->txr;
1352 	bool		more_tx, more_rx;
1353 
1354 	/* Protect against spurious interrupts */
1355 	if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING))
1356 		return;
1357 
1358 	++que->irqs;
1359 
1360 	more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1361 
1362 	IXL_TX_LOCK(txr);
1363 	more_tx = ixl_txeof(que);
1364 	/*
1365 	** Make certain that if the stack
1366 	** has anything queued the task gets
1367 	** scheduled to handle it.
1368 	*/
1369 	if (!drbr_empty(vsi->ifp, txr->br))
1370 		more_tx = 1;
1371 	IXL_TX_UNLOCK(txr);
1372 
1373 	ixl_set_queue_rx_itr(que);
1374 	ixl_set_queue_tx_itr(que);
1375 
1376 	if (more_tx || more_rx)
1377 		taskqueue_enqueue(que->tq, &que->task);
1378 	else
1379 		ixl_enable_queue(hw, que->me);
1380 
1381 	return;
1382 }
1383 
1384 
1385 /*********************************************************************
1386  *
1387  *  MSIX Admin Queue Interrupt Service routine
1388  *
1389  **********************************************************************/
1390 static void
1391 ixl_msix_adminq(void *arg)
1392 {
1393 	struct ixl_pf	*pf = arg;
1394 	struct i40e_hw	*hw = &pf->hw;
1395 	u32		reg, mask;
1396 
1397 	++pf->admin_irq;
1398 
1399 	reg = rd32(hw, I40E_PFINT_ICR0);
1400 	mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1401 
1402 	/* Check on the cause */
1403 	if (reg & I40E_PFINT_ICR0_ADMINQ_MASK)
1404 		mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
1405 
1406 	if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
1407 		ixl_handle_mdd_event(pf);
1408 		mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
1409 	}
1410 
1411 #ifdef PCI_IOV
1412 	if (reg & I40E_PFINT_ICR0_VFLR_MASK) {
1413 		mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
1414 		taskqueue_enqueue(pf->tq, &pf->vflr_task);
1415 	}
1416 #endif
1417 
1418 	reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1419 	reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1420 	wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1421 
1422 	taskqueue_enqueue(pf->tq, &pf->adminq);
1423 	return;
1424 }
1425 
1426 /*********************************************************************
1427  *
1428  *  Media Ioctl callback
1429  *
1430  *  This routine is called whenever the user queries the status of
1431  *  the interface using ifconfig.
1432  *
1433  **********************************************************************/
1434 static void
1435 ixl_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1436 {
1437 	struct ixl_vsi	*vsi = ifp->if_softc;
1438 	struct ixl_pf	*pf = vsi->back;
1439 	struct i40e_hw  *hw = &pf->hw;
1440 
1441 	INIT_DEBUGOUT("ixl_media_status: begin");
1442 	IXL_PF_LOCK(pf);
1443 
1444 	hw->phy.get_link_info = TRUE;
1445 	i40e_get_link_status(hw, &pf->link_up);
1446 	ixl_update_link_status(pf);
1447 
1448 	ifmr->ifm_status = IFM_AVALID;
1449 	ifmr->ifm_active = IFM_ETHER;
1450 
1451 	if (!pf->link_up) {
1452 		IXL_PF_UNLOCK(pf);
1453 		return;
1454 	}
1455 
1456 	ifmr->ifm_status |= IFM_ACTIVE;
1457 	/* Hardware is always full-duplex */
1458 	ifmr->ifm_active |= IFM_FDX;
1459 
1460 	switch (hw->phy.link_info.phy_type) {
1461 		/* 100 M */
1462 		case I40E_PHY_TYPE_100BASE_TX:
1463 			ifmr->ifm_active |= IFM_100_TX;
1464 			break;
1465 		/* 1 G */
1466 		case I40E_PHY_TYPE_1000BASE_T:
1467 			ifmr->ifm_active |= IFM_1000_T;
1468 			break;
1469 		case I40E_PHY_TYPE_1000BASE_SX:
1470 			ifmr->ifm_active |= IFM_1000_SX;
1471 			break;
1472 		case I40E_PHY_TYPE_1000BASE_LX:
1473 			ifmr->ifm_active |= IFM_1000_LX;
1474 			break;
1475 		/* 10 G */
1476 		case I40E_PHY_TYPE_10GBASE_SFPP_CU:
1477 			ifmr->ifm_active |= IFM_10G_TWINAX;
1478 			break;
1479 		case I40E_PHY_TYPE_10GBASE_SR:
1480 			ifmr->ifm_active |= IFM_10G_SR;
1481 			break;
1482 		case I40E_PHY_TYPE_10GBASE_LR:
1483 			ifmr->ifm_active |= IFM_10G_LR;
1484 			break;
1485 		case I40E_PHY_TYPE_10GBASE_T:
1486 			ifmr->ifm_active |= IFM_10G_T;
1487 			break;
1488 		/* 40 G */
1489 		case I40E_PHY_TYPE_40GBASE_CR4:
1490 		case I40E_PHY_TYPE_40GBASE_CR4_CU:
1491 			ifmr->ifm_active |= IFM_40G_CR4;
1492 			break;
1493 		case I40E_PHY_TYPE_40GBASE_SR4:
1494 			ifmr->ifm_active |= IFM_40G_SR4;
1495 			break;
1496 		case I40E_PHY_TYPE_40GBASE_LR4:
1497 			ifmr->ifm_active |= IFM_40G_LR4;
1498 			break;
1499 #ifndef IFM_ETH_XTYPE
1500 		case I40E_PHY_TYPE_1000BASE_KX:
1501 			ifmr->ifm_active |= IFM_1000_CX;
1502 			break;
1503 		case I40E_PHY_TYPE_10GBASE_CR1_CU:
1504 		case I40E_PHY_TYPE_10GBASE_CR1:
1505 			ifmr->ifm_active |= IFM_10G_TWINAX;
1506 			break;
1507 		case I40E_PHY_TYPE_10GBASE_KX4:
1508 			ifmr->ifm_active |= IFM_10G_CX4;
1509 			break;
1510 		case I40E_PHY_TYPE_10GBASE_KR:
1511 			ifmr->ifm_active |= IFM_10G_SR;
1512 			break;
1513 		case I40E_PHY_TYPE_40GBASE_KR4:
1514 		case I40E_PHY_TYPE_XLPPI:
1515 			ifmr->ifm_active |= IFM_40G_SR4;
1516 			break;
1517 #else
1518 		case I40E_PHY_TYPE_1000BASE_KX:
1519 			ifmr->ifm_active |= IFM_1000_KX;
1520 			break;
1521 		/* ERJ: What's the difference between these? */
1522 		case I40E_PHY_TYPE_10GBASE_CR1_CU:
1523 		case I40E_PHY_TYPE_10GBASE_CR1:
1524 			ifmr->ifm_active |= IFM_10G_CR1;
1525 			break;
1526 		case I40E_PHY_TYPE_10GBASE_KX4:
1527 			ifmr->ifm_active |= IFM_10G_KX4;
1528 			break;
1529 		case I40E_PHY_TYPE_10GBASE_KR:
1530 			ifmr->ifm_active |= IFM_10G_KR;
1531 			break;
1532 		case I40E_PHY_TYPE_20GBASE_KR2:
1533 			ifmr->ifm_active |= IFM_20G_KR2;
1534 			break;
1535 		case I40E_PHY_TYPE_40GBASE_KR4:
1536 			ifmr->ifm_active |= IFM_40G_KR4;
1537 			break;
1538 		case I40E_PHY_TYPE_XLPPI:
1539 			ifmr->ifm_active |= IFM_40G_XLPPI;
1540 			break;
1541 #endif
1542 		default:
1543 			ifmr->ifm_active |= IFM_UNKNOWN;
1544 			break;
1545 	}
1546 	/* Report flow control status as well */
1547 	if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX)
1548 		ifmr->ifm_active |= IFM_ETH_TXPAUSE;
1549 	if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX)
1550 		ifmr->ifm_active |= IFM_ETH_RXPAUSE;
1551 
1552 	IXL_PF_UNLOCK(pf);
1553 
1554 	return;
1555 }
1556 
1557 /*********************************************************************
1558  *
1559  *  Media Ioctl callback
1560  *
1561  *  This routine is called when the user changes speed/duplex using
1562  *  media/mediopt option with ifconfig.
1563  *
1564  **********************************************************************/
1565 static int
1566 ixl_media_change(struct ifnet * ifp)
1567 {
1568 	struct ixl_vsi *vsi = ifp->if_softc;
1569 	struct ifmedia *ifm = &vsi->media;
1570 
1571 	INIT_DEBUGOUT("ixl_media_change: begin");
1572 
1573 	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1574 		return (EINVAL);
1575 
1576 	if_printf(ifp, "Media change is currently not supported.\n");
1577 
1578 	return (ENODEV);
1579 }
1580 
1581 
1582 #ifdef IXL_FDIR
1583 /*
1584 ** ATR: Application Targetted Receive - creates a filter
1585 **	based on TX flow info that will keep the receive
1586 **	portion of the flow on the same queue. Based on the
1587 **	implementation this is only available for TCP connections
1588 */
1589 void
1590 ixl_atr(struct ixl_queue *que, struct tcphdr *th, int etype)
1591 {
1592 	struct ixl_vsi			*vsi = que->vsi;
1593 	struct tx_ring			*txr = &que->txr;
1594 	struct i40e_filter_program_desc	*FDIR;
1595 	u32				ptype, dtype;
1596 	int				idx;
1597 
1598 	/* check if ATR is enabled and sample rate */
1599 	if ((!ixl_enable_fdir) || (!txr->atr_rate))
1600 		return;
1601 	/*
1602 	** We sample all TCP SYN/FIN packets,
1603 	** or at the selected sample rate
1604 	*/
1605 	txr->atr_count++;
1606 	if (((th->th_flags & (TH_FIN | TH_SYN)) == 0) &&
1607 	    (txr->atr_count < txr->atr_rate))
1608                 return;
1609 	txr->atr_count = 0;
1610 
1611 	/* Get a descriptor to use */
1612 	idx = txr->next_avail;
1613 	FDIR = (struct i40e_filter_program_desc *) &txr->base[idx];
1614 	if (++idx == que->num_desc)
1615 		idx = 0;
1616 	txr->avail--;
1617 	txr->next_avail = idx;
1618 
1619 	ptype = (que->me << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1620 	    I40E_TXD_FLTR_QW0_QINDEX_MASK;
1621 
1622 	ptype |= (etype == ETHERTYPE_IP) ?
1623 	    (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
1624 	    I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
1625 	    (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
1626 	    I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
1627 
1628 	ptype |= vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
1629 
1630 	dtype = I40E_TX_DESC_DTYPE_FILTER_PROG;
1631 
1632 	/*
1633 	** We use the TCP TH_FIN as a trigger to remove
1634 	** the filter, otherwise its an update.
1635 	*/
1636 	dtype |= (th->th_flags & TH_FIN) ?
1637 	    (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1638 	    I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
1639 	    (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1640 	    I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1641 
1642 	dtype |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
1643 	    I40E_TXD_FLTR_QW1_DEST_SHIFT;
1644 
1645 	dtype |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
1646 	    I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
1647 
1648 	FDIR->qindex_flex_ptype_vsi = htole32(ptype);
1649 	FDIR->dtype_cmd_cntindex = htole32(dtype);
1650 	return;
1651 }
1652 #endif
1653 
1654 
1655 static void
1656 ixl_set_promisc(struct ixl_vsi *vsi)
1657 {
1658 	struct ifnet	*ifp = vsi->ifp;
1659 	struct i40e_hw	*hw = vsi->hw;
1660 	int		err, mcnt = 0;
1661 	bool		uni = FALSE, multi = FALSE;
1662 
1663 	if (ifp->if_flags & IFF_ALLMULTI)
1664                 multi = TRUE;
1665 	else { /* Need to count the multicast addresses */
1666 		struct  ifmultiaddr *ifma;
1667 		if_maddr_rlock(ifp);
1668 		TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1669                         if (ifma->ifma_addr->sa_family != AF_LINK)
1670                                 continue;
1671                         if (mcnt == MAX_MULTICAST_ADDR)
1672                                 break;
1673                         mcnt++;
1674 		}
1675 		if_maddr_runlock(ifp);
1676 	}
1677 
1678 	if (mcnt >= MAX_MULTICAST_ADDR)
1679                 multi = TRUE;
1680         if (ifp->if_flags & IFF_PROMISC)
1681 		uni = TRUE;
1682 
1683 	err = i40e_aq_set_vsi_unicast_promiscuous(hw,
1684 	    vsi->seid, uni, NULL);
1685 	err = i40e_aq_set_vsi_multicast_promiscuous(hw,
1686 	    vsi->seid, multi, NULL);
1687 	return;
1688 }
1689 
1690 /*********************************************************************
1691  * 	Filter Routines
1692  *
1693  *	Routines for multicast and vlan filter management.
1694  *
1695  *********************************************************************/
1696 static void
1697 ixl_add_multi(struct ixl_vsi *vsi)
1698 {
1699 	struct	ifmultiaddr	*ifma;
1700 	struct ifnet		*ifp = vsi->ifp;
1701 	struct i40e_hw		*hw = vsi->hw;
1702 	int			mcnt = 0, flags;
1703 
1704 	IOCTL_DEBUGOUT("ixl_add_multi: begin");
1705 
1706 	if_maddr_rlock(ifp);
1707 	/*
1708 	** First just get a count, to decide if we
1709 	** we simply use multicast promiscuous.
1710 	*/
1711 	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1712 		if (ifma->ifma_addr->sa_family != AF_LINK)
1713 			continue;
1714 		mcnt++;
1715 	}
1716 	if_maddr_runlock(ifp);
1717 
1718 	if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) {
1719 		/* delete existing MC filters */
1720 		ixl_del_hw_filters(vsi, mcnt);
1721 		i40e_aq_set_vsi_multicast_promiscuous(hw,
1722 		    vsi->seid, TRUE, NULL);
1723 		return;
1724 	}
1725 
1726 	mcnt = 0;
1727 	if_maddr_rlock(ifp);
1728 	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1729 		if (ifma->ifma_addr->sa_family != AF_LINK)
1730 			continue;
1731 		ixl_add_mc_filter(vsi,
1732 		    (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr));
1733 		mcnt++;
1734 	}
1735 	if_maddr_runlock(ifp);
1736 	if (mcnt > 0) {
1737 		flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC);
1738 		ixl_add_hw_filters(vsi, flags, mcnt);
1739 	}
1740 
1741 	IOCTL_DEBUGOUT("ixl_add_multi: end");
1742 	return;
1743 }
1744 
1745 static void
1746 ixl_del_multi(struct ixl_vsi *vsi)
1747 {
1748 	struct ifnet		*ifp = vsi->ifp;
1749 	struct ifmultiaddr	*ifma;
1750 	struct ixl_mac_filter	*f;
1751 	int			mcnt = 0;
1752 	bool		match = FALSE;
1753 
1754 	IOCTL_DEBUGOUT("ixl_del_multi: begin");
1755 
1756 	/* Search for removed multicast addresses */
1757 	if_maddr_rlock(ifp);
1758 	SLIST_FOREACH(f, &vsi->ftl, next) {
1759 		if ((f->flags & IXL_FILTER_USED) && (f->flags & IXL_FILTER_MC)) {
1760 			match = FALSE;
1761 			TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1762 				if (ifma->ifma_addr->sa_family != AF_LINK)
1763 					continue;
1764 				u8 *mc_addr = (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
1765 				if (cmp_etheraddr(f->macaddr, mc_addr)) {
1766 					match = TRUE;
1767 					break;
1768 				}
1769 			}
1770 			if (match == FALSE) {
1771 				f->flags |= IXL_FILTER_DEL;
1772 				mcnt++;
1773 			}
1774 		}
1775 	}
1776 	if_maddr_runlock(ifp);
1777 
1778 	if (mcnt > 0)
1779 		ixl_del_hw_filters(vsi, mcnt);
1780 }
1781 
1782 
1783 /*********************************************************************
1784  *  Timer routine
1785  *
1786  *  This routine checks for link status,updates statistics,
1787  *  and runs the watchdog check.
1788  *
1789  **********************************************************************/
1790 
1791 static void
1792 ixl_local_timer(void *arg)
1793 {
1794 	struct ixl_pf		*pf = arg;
1795 	struct i40e_hw		*hw = &pf->hw;
1796 	struct ixl_vsi		*vsi = &pf->vsi;
1797 	struct ixl_queue	*que = vsi->queues;
1798 	device_t		dev = pf->dev;
1799 	int			hung = 0;
1800 	u32			mask;
1801 
1802 	mtx_assert(&pf->pf_mtx, MA_OWNED);
1803 
1804 	/* Fire off the adminq task */
1805 	taskqueue_enqueue(pf->tq, &pf->adminq);
1806 
1807 	/* Update stats */
1808 	ixl_update_stats_counters(pf);
1809 
1810 	/*
1811 	** Check status of the queues
1812 	*/
1813 	mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK |
1814 		I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK);
1815 
1816 	for (int i = 0; i < vsi->num_queues; i++,que++) {
1817 		/* Any queues with outstanding work get a sw irq */
1818 		if (que->busy)
1819 			wr32(hw, I40E_PFINT_DYN_CTLN(que->me), mask);
1820 		/*
1821 		** Each time txeof runs without cleaning, but there
1822 		** are uncleaned descriptors it increments busy. If
1823 		** we get to 5 we declare it hung.
1824 		*/
1825 		if (que->busy == IXL_QUEUE_HUNG) {
1826 			++hung;
1827 			/* Mark the queue as inactive */
1828 			vsi->active_queues &= ~((u64)1 << que->me);
1829 			continue;
1830 		} else {
1831 			/* Check if we've come back from hung */
1832 			if ((vsi->active_queues & ((u64)1 << que->me)) == 0)
1833 				vsi->active_queues |= ((u64)1 << que->me);
1834 		}
1835 		if (que->busy >= IXL_MAX_TX_BUSY) {
1836 #ifdef IXL_DEBUG
1837 			device_printf(dev,"Warning queue %d "
1838 			    "appears to be hung!\n", i);
1839 #endif
1840 			que->busy = IXL_QUEUE_HUNG;
1841 			++hung;
1842 		}
1843 	}
1844 	/* Only reinit if all queues show hung */
1845 	if (hung == vsi->num_queues)
1846 		goto hung;
1847 
1848 	callout_reset(&pf->timer, hz, ixl_local_timer, pf);
1849 	return;
1850 
1851 hung:
1852 	device_printf(dev, "Local Timer: HANG DETECT - Resetting!!\n");
1853 	ixl_init_locked(pf);
1854 }
1855 
1856 /*
1857 ** Note: this routine updates the OS on the link state
1858 **	the real check of the hardware only happens with
1859 **	a link interrupt.
1860 */
1861 static void
1862 ixl_update_link_status(struct ixl_pf *pf)
1863 {
1864 	struct ixl_vsi		*vsi = &pf->vsi;
1865 	struct i40e_hw		*hw = &pf->hw;
1866 	struct ifnet		*ifp = vsi->ifp;
1867 	device_t		dev = pf->dev;
1868 
1869 	if (pf->link_up){
1870 		if (vsi->link_active == FALSE) {
1871 			pf->fc = hw->fc.current_mode;
1872 			if (bootverbose) {
1873 				device_printf(dev,"Link is up %d Gbps %s,"
1874 				    " Flow Control: %s\n",
1875 				    ((pf->link_speed ==
1876 				    I40E_LINK_SPEED_40GB)? 40:10),
1877 				    "Full Duplex", ixl_fc_string[pf->fc]);
1878 			}
1879 			vsi->link_active = TRUE;
1880 			/*
1881 			** Warn user if link speed on NPAR enabled
1882 			** partition is not at least 10GB
1883 			*/
1884 			if (hw->func_caps.npar_enable &&
1885 			   (hw->phy.link_info.link_speed ==
1886 			   I40E_LINK_SPEED_1GB ||
1887 			   hw->phy.link_info.link_speed ==
1888 			   I40E_LINK_SPEED_100MB))
1889 				device_printf(dev, "The partition detected"
1890 				    "link speed that is less than 10Gbps\n");
1891 			if_link_state_change(ifp, LINK_STATE_UP);
1892 		}
1893 	} else { /* Link down */
1894 		if (vsi->link_active == TRUE) {
1895 			if (bootverbose)
1896 				device_printf(dev,"Link is Down\n");
1897 			if_link_state_change(ifp, LINK_STATE_DOWN);
1898 			vsi->link_active = FALSE;
1899 		}
1900 	}
1901 
1902 	return;
1903 }
1904 
1905 /*********************************************************************
1906  *
1907  *  This routine disables all traffic on the adapter by issuing a
1908  *  global reset on the MAC and deallocates TX/RX buffers.
1909  *
1910  **********************************************************************/
1911 
1912 static void
1913 ixl_stop(struct ixl_pf *pf)
1914 {
1915 	struct ixl_vsi	*vsi = &pf->vsi;
1916 	struct ifnet	*ifp = vsi->ifp;
1917 
1918 	mtx_assert(&pf->pf_mtx, MA_OWNED);
1919 
1920 	INIT_DEBUGOUT("ixl_stop: begin\n");
1921 	if (pf->num_vfs == 0)
1922 		ixl_disable_intr(vsi);
1923 	else
1924 		ixl_disable_rings_intr(vsi);
1925 	ixl_disable_rings(vsi);
1926 
1927 	/* Tell the stack that the interface is no longer active */
1928 	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1929 
1930 	/* Stop the local timer */
1931 	callout_stop(&pf->timer);
1932 
1933 	return;
1934 }
1935 
1936 
1937 /*********************************************************************
1938  *
1939  *  Setup MSIX Interrupt resources and handlers for the VSI
1940  *
1941  **********************************************************************/
1942 static int
1943 ixl_assign_vsi_legacy(struct ixl_pf *pf)
1944 {
1945 	device_t        dev = pf->dev;
1946 	struct 		ixl_vsi *vsi = &pf->vsi;
1947 	struct		ixl_queue *que = vsi->queues;
1948 	int 		error, rid = 0;
1949 
1950 	if (pf->msix == 1)
1951 		rid = 1;
1952 	pf->res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
1953 	    &rid, RF_SHAREABLE | RF_ACTIVE);
1954 	if (pf->res == NULL) {
1955 		device_printf(dev,"Unable to allocate"
1956 		    " bus resource: vsi legacy/msi interrupt\n");
1957 		return (ENXIO);
1958 	}
1959 
1960 	/* Set the handler function */
1961 	error = bus_setup_intr(dev, pf->res,
1962 	    INTR_TYPE_NET | INTR_MPSAFE, NULL,
1963 	    ixl_intr, pf, &pf->tag);
1964 	if (error) {
1965 		pf->res = NULL;
1966 		device_printf(dev, "Failed to register legacy/msi handler");
1967 		return (error);
1968 	}
1969 	bus_describe_intr(dev, pf->res, pf->tag, "irq0");
1970 	TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
1971 	TASK_INIT(&que->task, 0, ixl_handle_que, que);
1972 	que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
1973 	    taskqueue_thread_enqueue, &que->tq);
1974 	taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
1975 	    device_get_nameunit(dev));
1976 	TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
1977 
1978 #ifdef PCI_IOV
1979 	TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf);
1980 #endif
1981 
1982 	pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT,
1983 	    taskqueue_thread_enqueue, &pf->tq);
1984 	taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq",
1985 	    device_get_nameunit(dev));
1986 
1987 	return (0);
1988 }
1989 
1990 
1991 /*********************************************************************
1992  *
1993  *  Setup MSIX Interrupt resources and handlers for the VSI
1994  *
1995  **********************************************************************/
1996 static int
1997 ixl_assign_vsi_msix(struct ixl_pf *pf)
1998 {
1999 	device_t	dev = pf->dev;
2000 	struct 		ixl_vsi *vsi = &pf->vsi;
2001 	struct 		ixl_queue *que = vsi->queues;
2002 	struct		tx_ring	 *txr;
2003 	int 		error, rid, vector = 0;
2004 #ifdef	RSS
2005 	cpuset_t cpu_mask;
2006 #endif
2007 
2008 	/* Admin Que is vector 0*/
2009 	rid = vector + 1;
2010 	pf->res = bus_alloc_resource_any(dev,
2011     	    SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2012 	if (!pf->res) {
2013 		device_printf(dev,"Unable to allocate"
2014     	    " bus resource: Adminq interrupt [%d]\n", rid);
2015 		return (ENXIO);
2016 	}
2017 	/* Set the adminq vector and handler */
2018 	error = bus_setup_intr(dev, pf->res,
2019 	    INTR_TYPE_NET | INTR_MPSAFE, NULL,
2020 	    ixl_msix_adminq, pf, &pf->tag);
2021 	if (error) {
2022 		pf->res = NULL;
2023 		device_printf(dev, "Failed to register Admin que handler");
2024 		return (error);
2025 	}
2026 	bus_describe_intr(dev, pf->res, pf->tag, "aq");
2027 	pf->admvec = vector;
2028 	/* Tasklet for Admin Queue */
2029 	TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
2030 
2031 #ifdef PCI_IOV
2032 	TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf);
2033 #endif
2034 
2035 	pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT,
2036 	    taskqueue_thread_enqueue, &pf->tq);
2037 	taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq",
2038 	    device_get_nameunit(pf->dev));
2039 	++vector;
2040 
2041 	/* Now set up the stations */
2042 	for (int i = 0; i < vsi->num_queues; i++, vector++, que++) {
2043 		int cpu_id = i;
2044 		rid = vector + 1;
2045 		txr = &que->txr;
2046 		que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2047 		    RF_SHAREABLE | RF_ACTIVE);
2048 		if (que->res == NULL) {
2049 			device_printf(dev,"Unable to allocate"
2050 		    	    " bus resource: que interrupt [%d]\n", vector);
2051 			return (ENXIO);
2052 		}
2053 		/* Set the handler function */
2054 		error = bus_setup_intr(dev, que->res,
2055 		    INTR_TYPE_NET | INTR_MPSAFE, NULL,
2056 		    ixl_msix_que, que, &que->tag);
2057 		if (error) {
2058 			que->res = NULL;
2059 			device_printf(dev, "Failed to register que handler");
2060 			return (error);
2061 		}
2062 		bus_describe_intr(dev, que->res, que->tag, "q%d", i);
2063 		/* Bind the vector to a CPU */
2064 #ifdef RSS
2065 		cpu_id = rss_getcpu(i % rss_getnumbuckets());
2066 #endif
2067 		bus_bind_intr(dev, que->res, cpu_id);
2068 		que->msix = vector;
2069 		TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
2070 		TASK_INIT(&que->task, 0, ixl_handle_que, que);
2071 		que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
2072 		    taskqueue_thread_enqueue, &que->tq);
2073 #ifdef RSS
2074 		CPU_SETOF(cpu_id, &cpu_mask);
2075 		taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET,
2076 		    &cpu_mask, "%s (bucket %d)",
2077 		    device_get_nameunit(dev), cpu_id);
2078 #else
2079 		taskqueue_start_threads(&que->tq, 1, PI_NET,
2080 		    "%s que", device_get_nameunit(dev));
2081 #endif
2082 	}
2083 
2084 	return (0);
2085 }
2086 
2087 
2088 /*
2089  * Allocate MSI/X vectors
2090  */
2091 static int
2092 ixl_init_msix(struct ixl_pf *pf)
2093 {
2094 	device_t dev = pf->dev;
2095 	int rid, want, vectors, queues, available;
2096 
2097 	/* Override by tuneable */
2098 	if (ixl_enable_msix == 0)
2099 		goto msi;
2100 
2101 	/*
2102 	** When used in a virtualized environment
2103 	** PCI BUSMASTER capability may not be set
2104 	** so explicity set it here and rewrite
2105 	** the ENABLE in the MSIX control register
2106 	** at this point to cause the host to
2107 	** successfully initialize us.
2108 	*/
2109 	{
2110 		u16 pci_cmd_word;
2111 		int msix_ctrl;
2112 		pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2113 		pci_cmd_word |= PCIM_CMD_BUSMASTEREN;
2114 		pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2);
2115 		pci_find_cap(dev, PCIY_MSIX, &rid);
2116 		rid += PCIR_MSIX_CTRL;
2117 		msix_ctrl = pci_read_config(dev, rid, 2);
2118 		msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
2119 		pci_write_config(dev, rid, msix_ctrl, 2);
2120 	}
2121 
2122 	/* First try MSI/X */
2123 	rid = PCIR_BAR(IXL_BAR);
2124 	pf->msix_mem = bus_alloc_resource_any(dev,
2125 	    SYS_RES_MEMORY, &rid, RF_ACTIVE);
2126        	if (!pf->msix_mem) {
2127 		/* May not be enabled */
2128 		device_printf(pf->dev,
2129 		    "Unable to map MSIX table \n");
2130 		goto msi;
2131 	}
2132 
2133 	available = pci_msix_count(dev);
2134 	if (available == 0) { /* system has msix disabled */
2135 		bus_release_resource(dev, SYS_RES_MEMORY,
2136 		    rid, pf->msix_mem);
2137 		pf->msix_mem = NULL;
2138 		goto msi;
2139 	}
2140 
2141 	/* Figure out a reasonable auto config value */
2142 	queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus;
2143 
2144 	/* Override with hardcoded value if sane */
2145 	if ((ixl_max_queues != 0) && (ixl_max_queues <= queues))
2146 		queues = ixl_max_queues;
2147 
2148 #ifdef  RSS
2149 	/* If we're doing RSS, clamp at the number of RSS buckets */
2150 	if (queues > rss_getnumbuckets())
2151 		queues = rss_getnumbuckets();
2152 #endif
2153 
2154 	/*
2155 	** Want one vector (RX/TX pair) per queue
2156 	** plus an additional for the admin queue.
2157 	*/
2158 	want = queues + 1;
2159 	if (want <= available)	/* Have enough */
2160 		vectors = want;
2161 	else {
2162                	device_printf(pf->dev,
2163 		    "MSIX Configuration Problem, "
2164 		    "%d vectors available but %d wanted!\n",
2165 		    available, want);
2166 		return (0); /* Will go to Legacy setup */
2167 	}
2168 
2169 	if (pci_alloc_msix(dev, &vectors) == 0) {
2170                	device_printf(pf->dev,
2171 		    "Using MSIX interrupts with %d vectors\n", vectors);
2172 		pf->msix = vectors;
2173 		pf->vsi.num_queues = queues;
2174 #ifdef RSS
2175 		/*
2176 		 * If we're doing RSS, the number of queues needs to
2177 		 * match the number of RSS buckets that are configured.
2178 		 *
2179 		 * + If there's more queues than RSS buckets, we'll end
2180 		 *   up with queues that get no traffic.
2181 		 *
2182 		 * + If there's more RSS buckets than queues, we'll end
2183 		 *   up having multiple RSS buckets map to the same queue,
2184 		 *   so there'll be some contention.
2185 		 */
2186 		if (queues != rss_getnumbuckets()) {
2187 			device_printf(dev,
2188 			    "%s: queues (%d) != RSS buckets (%d)"
2189 			    "; performance will be impacted.\n",
2190 			    __func__, queues, rss_getnumbuckets());
2191 		}
2192 #endif
2193 		return (vectors);
2194 	}
2195 msi:
2196        	vectors = pci_msi_count(dev);
2197 	pf->vsi.num_queues = 1;
2198 	pf->msix = 1;
2199 	ixl_max_queues = 1;
2200 	ixl_enable_msix = 0;
2201        	if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0)
2202                	device_printf(pf->dev,"Using an MSI interrupt\n");
2203 	else {
2204 		pf->msix = 0;
2205                	device_printf(pf->dev,"Using a Legacy interrupt\n");
2206 	}
2207 	return (vectors);
2208 }
2209 
2210 
2211 /*
2212  * Plumb MSI/X vectors
2213  */
2214 static void
2215 ixl_configure_msix(struct ixl_pf *pf)
2216 {
2217 	struct i40e_hw	*hw = &pf->hw;
2218 	struct ixl_vsi *vsi = &pf->vsi;
2219 	u32		reg;
2220 	u16		vector = 1;
2221 
2222 	/* First set up the adminq - vector 0 */
2223 	wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
2224 	rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
2225 
2226 	reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
2227 	    I40E_PFINT_ICR0_ENA_GRST_MASK |
2228 	    I40E_PFINT_ICR0_HMC_ERR_MASK |
2229 	    I40E_PFINT_ICR0_ENA_ADMINQ_MASK |
2230 	    I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
2231 	    I40E_PFINT_ICR0_ENA_VFLR_MASK |
2232 	    I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK;
2233 	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2234 
2235 	wr32(hw, I40E_PFINT_LNKLST0, 0x7FF);
2236 	wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR), 0x003E);
2237 
2238 	wr32(hw, I40E_PFINT_DYN_CTL0,
2239 	    I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
2240 	    I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
2241 
2242 	wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2243 
2244 	/* Next configure the queues */
2245 	for (int i = 0; i < vsi->num_queues; i++, vector++) {
2246 		wr32(hw, I40E_PFINT_DYN_CTLN(i), i);
2247 		wr32(hw, I40E_PFINT_LNKLSTN(i), i);
2248 
2249 		reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
2250 		(IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
2251 		(vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
2252 		(i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
2253 		(I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
2254 		wr32(hw, I40E_QINT_RQCTL(i), reg);
2255 
2256 		reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
2257 		(IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
2258 		(vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
2259 		((i+1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
2260 		(I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2261 		if (i == (vsi->num_queues - 1))
2262 			reg |= (IXL_QUEUE_EOL
2263 			    << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2264 		wr32(hw, I40E_QINT_TQCTL(i), reg);
2265 	}
2266 }
2267 
2268 /*
2269  * Configure for MSI single vector operation
2270  */
2271 static void
2272 ixl_configure_legacy(struct ixl_pf *pf)
2273 {
2274 	struct i40e_hw	*hw = &pf->hw;
2275 	u32		reg;
2276 
2277 
2278 	wr32(hw, I40E_PFINT_ITR0(0), 0);
2279 	wr32(hw, I40E_PFINT_ITR0(1), 0);
2280 
2281 
2282 	/* Setup "other" causes */
2283 	reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK
2284 	    | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK
2285 	    | I40E_PFINT_ICR0_ENA_GRST_MASK
2286 	    | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK
2287 	    | I40E_PFINT_ICR0_ENA_GPIO_MASK
2288 	    | I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK
2289 	    | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK
2290 	    | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK
2291 	    | I40E_PFINT_ICR0_ENA_VFLR_MASK
2292 	    | I40E_PFINT_ICR0_ENA_ADMINQ_MASK
2293 	    ;
2294 	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2295 
2296 	/* SW_ITR_IDX = 0, but don't change INTENA */
2297 	wr32(hw, I40E_PFINT_DYN_CTL0,
2298 	    I40E_PFINT_DYN_CTLN_SW_ITR_INDX_MASK |
2299 	    I40E_PFINT_DYN_CTLN_INTENA_MSK_MASK);
2300 	/* SW_ITR_IDX = 0, OTHER_ITR_IDX = 0 */
2301 	wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2302 
2303 	/* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
2304 	wr32(hw, I40E_PFINT_LNKLST0, 0);
2305 
2306 	/* Associate the queue pair to the vector and enable the q int */
2307 	reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK
2308 	    | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)
2309 	    | (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2310 	wr32(hw, I40E_QINT_RQCTL(0), reg);
2311 
2312 	reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK
2313 	    | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)
2314 	    | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2315 	wr32(hw, I40E_QINT_TQCTL(0), reg);
2316 
2317 	/* Next enable the queue pair */
2318 	reg = rd32(hw, I40E_QTX_ENA(0));
2319 	reg |= I40E_QTX_ENA_QENA_REQ_MASK;
2320 	wr32(hw, I40E_QTX_ENA(0), reg);
2321 
2322 	reg = rd32(hw, I40E_QRX_ENA(0));
2323 	reg |= I40E_QRX_ENA_QENA_REQ_MASK;
2324 	wr32(hw, I40E_QRX_ENA(0), reg);
2325 }
2326 
2327 
2328 /*
2329  * Set the Initial ITR state
2330  */
2331 static void
2332 ixl_configure_itr(struct ixl_pf *pf)
2333 {
2334 	struct i40e_hw		*hw = &pf->hw;
2335 	struct ixl_vsi		*vsi = &pf->vsi;
2336 	struct ixl_queue	*que = vsi->queues;
2337 
2338 	vsi->rx_itr_setting = ixl_rx_itr;
2339 	if (ixl_dynamic_rx_itr)
2340 		vsi->rx_itr_setting |= IXL_ITR_DYNAMIC;
2341 	vsi->tx_itr_setting = ixl_tx_itr;
2342 	if (ixl_dynamic_tx_itr)
2343 		vsi->tx_itr_setting |= IXL_ITR_DYNAMIC;
2344 
2345 	for (int i = 0; i < vsi->num_queues; i++, que++) {
2346 		struct tx_ring	*txr = &que->txr;
2347 		struct rx_ring 	*rxr = &que->rxr;
2348 
2349 		wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i),
2350 		    vsi->rx_itr_setting);
2351 		rxr->itr = vsi->rx_itr_setting;
2352 		rxr->latency = IXL_AVE_LATENCY;
2353 		wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i),
2354 		    vsi->tx_itr_setting);
2355 		txr->itr = vsi->tx_itr_setting;
2356 		txr->latency = IXL_AVE_LATENCY;
2357 	}
2358 }
2359 
2360 
2361 static int
2362 ixl_allocate_pci_resources(struct ixl_pf *pf)
2363 {
2364 	int             rid;
2365 	device_t        dev = pf->dev;
2366 
2367 	rid = PCIR_BAR(0);
2368 	pf->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2369 	    &rid, RF_ACTIVE);
2370 
2371 	if (!(pf->pci_mem)) {
2372 		device_printf(dev,"Unable to allocate bus resource: memory\n");
2373 		return (ENXIO);
2374 	}
2375 
2376 	pf->osdep.mem_bus_space_tag =
2377 		rman_get_bustag(pf->pci_mem);
2378 	pf->osdep.mem_bus_space_handle =
2379 		rman_get_bushandle(pf->pci_mem);
2380 	pf->osdep.mem_bus_space_size = rman_get_size(pf->pci_mem);
2381 	pf->osdep.flush_reg = I40E_GLGEN_STAT;
2382 	pf->hw.hw_addr = (u8 *) &pf->osdep.mem_bus_space_handle;
2383 
2384 	pf->hw.back = &pf->osdep;
2385 
2386 	/*
2387 	** Now setup MSI or MSI/X, should
2388 	** return us the number of supported
2389 	** vectors. (Will be 1 for MSI)
2390 	*/
2391 	pf->msix = ixl_init_msix(pf);
2392 	return (0);
2393 }
2394 
2395 static void
2396 ixl_free_pci_resources(struct ixl_pf * pf)
2397 {
2398 	struct ixl_vsi		*vsi = &pf->vsi;
2399 	struct ixl_queue	*que = vsi->queues;
2400 	device_t		dev = pf->dev;
2401 	int			rid, memrid;
2402 
2403 	memrid = PCIR_BAR(IXL_BAR);
2404 
2405 	/* We may get here before stations are setup */
2406 	if ((!ixl_enable_msix) || (que == NULL))
2407 		goto early;
2408 
2409 	/*
2410 	**  Release all msix VSI resources:
2411 	*/
2412 	for (int i = 0; i < vsi->num_queues; i++, que++) {
2413 		rid = que->msix + 1;
2414 		if (que->tag != NULL) {
2415 			bus_teardown_intr(dev, que->res, que->tag);
2416 			que->tag = NULL;
2417 		}
2418 		if (que->res != NULL)
2419 			bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2420 	}
2421 
2422 early:
2423 	/* Clean the AdminQ interrupt last */
2424 	if (pf->admvec) /* we are doing MSIX */
2425 		rid = pf->admvec + 1;
2426 	else
2427 		(pf->msix != 0) ? (rid = 1):(rid = 0);
2428 
2429 	if (pf->tag != NULL) {
2430 		bus_teardown_intr(dev, pf->res, pf->tag);
2431 		pf->tag = NULL;
2432 	}
2433 	if (pf->res != NULL)
2434 		bus_release_resource(dev, SYS_RES_IRQ, rid, pf->res);
2435 
2436 	if (pf->msix)
2437 		pci_release_msi(dev);
2438 
2439 	if (pf->msix_mem != NULL)
2440 		bus_release_resource(dev, SYS_RES_MEMORY,
2441 		    memrid, pf->msix_mem);
2442 
2443 	if (pf->pci_mem != NULL)
2444 		bus_release_resource(dev, SYS_RES_MEMORY,
2445 		    PCIR_BAR(0), pf->pci_mem);
2446 
2447 	return;
2448 }
2449 
2450 static void
2451 ixl_add_ifmedia(struct ixl_vsi *vsi, u32 phy_type)
2452 {
2453 	/* Display supported media types */
2454 	if (phy_type & (1 << I40E_PHY_TYPE_100BASE_TX))
2455 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL);
2456 
2457 	if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_T))
2458 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL);
2459 	if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_SX))
2460 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
2461 	if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_LX))
2462 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL);
2463 
2464 	if (phy_type & (1 << I40E_PHY_TYPE_XAUI) ||
2465 	    phy_type & (1 << I40E_PHY_TYPE_XFI) ||
2466 	    phy_type & (1 << I40E_PHY_TYPE_10GBASE_SFPP_CU))
2467 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2468 
2469 	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_SR))
2470 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2471 	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_LR))
2472 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL);
2473 	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_T))
2474 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL);
2475 
2476 	if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4) ||
2477 	    phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4_CU) ||
2478 	    phy_type & (1 << I40E_PHY_TYPE_40GBASE_AOC) ||
2479 	    phy_type & (1 << I40E_PHY_TYPE_XLAUI) ||
2480 	    phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2481 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL);
2482 	if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_SR4))
2483 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
2484 	if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_LR4))
2485 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL);
2486 
2487 #ifndef IFM_ETH_XTYPE
2488 	if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX))
2489 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_CX, 0, NULL);
2490 
2491 	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) ||
2492 	    phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1) ||
2493 	    phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC) ||
2494 	    phy_type & (1 << I40E_PHY_TYPE_SFI))
2495 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2496 	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4))
2497 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CX4, 0, NULL);
2498 	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR))
2499 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2500 
2501 	if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2502 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
2503 	if (phy_type & (1 << I40E_PHY_TYPE_XLPPI))
2504 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL);
2505 #else
2506 	if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX))
2507 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_KX, 0, NULL);
2508 
2509 	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU)
2510 	    || phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1))
2511 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CR1, 0, NULL);
2512 	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC))
2513 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX_LONG, 0, NULL);
2514 	if (phy_type & (1 << I40E_PHY_TYPE_SFI))
2515 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SFI, 0, NULL);
2516 	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4))
2517 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KX4, 0, NULL);
2518 	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR))
2519 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KR, 0, NULL);
2520 
2521 	if (phy_type & (1 << I40E_PHY_TYPE_20GBASE_KR2))
2522 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_20G_KR2, 0, NULL);
2523 
2524 	if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2525 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_KR4, 0, NULL);
2526 	if (phy_type & (1 << I40E_PHY_TYPE_XLPPI))
2527 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_XLPPI, 0, NULL);
2528 #endif
2529 }
2530 
2531 /*********************************************************************
2532  *
2533  *  Setup networking device structure and register an interface.
2534  *
2535  **********************************************************************/
2536 static int
2537 ixl_setup_interface(device_t dev, struct ixl_vsi *vsi)
2538 {
2539 	struct ifnet		*ifp;
2540 	struct i40e_hw		*hw = vsi->hw;
2541 	struct ixl_queue	*que = vsi->queues;
2542 	struct i40e_aq_get_phy_abilities_resp abilities;
2543 	enum i40e_status_code aq_error = 0;
2544 
2545 	INIT_DEBUGOUT("ixl_setup_interface: begin");
2546 
2547 	ifp = vsi->ifp = if_alloc(IFT_ETHER);
2548 	if (ifp == NULL) {
2549 		device_printf(dev, "can not allocate ifnet structure\n");
2550 		return (-1);
2551 	}
2552 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2553 	ifp->if_mtu = ETHERMTU;
2554 	ifp->if_baudrate = 4000000000;  // ??
2555 	ifp->if_init = ixl_init;
2556 	ifp->if_softc = vsi;
2557 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2558 	ifp->if_ioctl = ixl_ioctl;
2559 
2560 #if __FreeBSD_version >= 1100036
2561 	if_setgetcounterfn(ifp, ixl_get_counter);
2562 #endif
2563 
2564 	ifp->if_transmit = ixl_mq_start;
2565 
2566 	ifp->if_qflush = ixl_qflush;
2567 
2568 	ifp->if_snd.ifq_maxlen = que->num_desc - 2;
2569 
2570 	vsi->max_frame_size =
2571 	    ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
2572 	    + ETHER_VLAN_ENCAP_LEN;
2573 
2574 	/*
2575 	 * Tell the upper layer(s) we support long frames.
2576 	 */
2577 	ifp->if_hdrlen = sizeof(struct ether_vlan_header);
2578 
2579 	ifp->if_capabilities |= IFCAP_HWCSUM;
2580 	ifp->if_capabilities |= IFCAP_HWCSUM_IPV6;
2581 	ifp->if_capabilities |= IFCAP_TSO;
2582 	ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2583 	ifp->if_capabilities |= IFCAP_LRO;
2584 
2585 	/* VLAN capabilties */
2586 	ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2587 			     |  IFCAP_VLAN_HWTSO
2588 			     |  IFCAP_VLAN_MTU
2589 			     |  IFCAP_VLAN_HWCSUM;
2590 	ifp->if_capenable = ifp->if_capabilities;
2591 
2592 	/*
2593 	** Don't turn this on by default, if vlans are
2594 	** created on another pseudo device (eg. lagg)
2595 	** then vlan events are not passed thru, breaking
2596 	** operation, but with HW FILTER off it works. If
2597 	** using vlans directly on the ixl driver you can
2598 	** enable this and get full hardware tag filtering.
2599 	*/
2600 	ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2601 
2602 	/*
2603 	 * Specify the media types supported by this adapter and register
2604 	 * callbacks to update media and link information
2605 	 */
2606 	ifmedia_init(&vsi->media, IFM_IMASK, ixl_media_change,
2607 		     ixl_media_status);
2608 
2609 	aq_error = i40e_aq_get_phy_capabilities(hw,
2610 	    FALSE, TRUE, &abilities, NULL);
2611 	/* May need delay to detect fiber correctly */
2612 	if (aq_error == I40E_ERR_UNKNOWN_PHY) {
2613 		i40e_msec_delay(200);
2614 		aq_error = i40e_aq_get_phy_capabilities(hw, FALSE,
2615 		    TRUE, &abilities, NULL);
2616 	}
2617 	if (aq_error) {
2618 		if (aq_error == I40E_ERR_UNKNOWN_PHY)
2619 			device_printf(dev, "Unknown PHY type detected!\n");
2620 		else
2621 			device_printf(dev,
2622 			    "Error getting supported media types, err %d,"
2623 			    " AQ error %d\n", aq_error, hw->aq.asq_last_status);
2624 		return (0);
2625 	}
2626 
2627 	ixl_add_ifmedia(vsi, abilities.phy_type);
2628 
2629 	/* Use autoselect media by default */
2630 	ifmedia_add(&vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2631 	ifmedia_set(&vsi->media, IFM_ETHER | IFM_AUTO);
2632 
2633 	ether_ifattach(ifp, hw->mac.addr);
2634 
2635 	return (0);
2636 }
2637 
2638 /*
2639 ** Run when the Admin Queue gets a
2640 ** link transition interrupt.
2641 */
2642 static void
2643 ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e)
2644 {
2645 	struct i40e_hw	*hw = &pf->hw;
2646 	struct i40e_aqc_get_link_status *status =
2647 	    (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
2648 	bool check;
2649 
2650 	hw->phy.get_link_info = TRUE;
2651 	i40e_get_link_status(hw, &check);
2652 	pf->link_up = check;
2653 #ifdef IXL_DEBUG
2654 	printf("Link is %s\n", check ? "up":"down");
2655 #endif
2656 	/* Report if Unqualified modules are found */
2657 	if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
2658 	    (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
2659 	    (!(status->link_info & I40E_AQ_LINK_UP)))
2660 		device_printf(pf->dev, "Link failed because "
2661 		    "an unqualified module was detected\n");
2662 
2663 	return;
2664 }
2665 
2666 /*********************************************************************
2667  *
2668  *  Get Firmware Switch configuration
2669  *	- this will need to be more robust when more complex
2670  *	  switch configurations are enabled.
2671  *
2672  **********************************************************************/
2673 static int
2674 ixl_switch_config(struct ixl_pf *pf)
2675 {
2676 	struct i40e_hw	*hw = &pf->hw;
2677 	struct ixl_vsi	*vsi = &pf->vsi;
2678 	device_t 	dev = vsi->dev;
2679 	struct i40e_aqc_get_switch_config_resp *sw_config;
2680 	u8	aq_buf[I40E_AQ_LARGE_BUF];
2681 	int	ret;
2682 	u16	next = 0;
2683 
2684 	memset(&aq_buf, 0, sizeof(aq_buf));
2685 	sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
2686 	ret = i40e_aq_get_switch_config(hw, sw_config,
2687 	    sizeof(aq_buf), &next, NULL);
2688 	if (ret) {
2689 		device_printf(dev,"aq_get_switch_config failed (ret=%d)!!\n",
2690 		    ret);
2691 		return (ret);
2692 	}
2693 #ifdef IXL_DEBUG
2694 	device_printf(dev,
2695 	    "Switch config: header reported: %d in structure, %d total\n",
2696     	    sw_config->header.num_reported, sw_config->header.num_total);
2697 	for (int i = 0; i < sw_config->header.num_reported; i++) {
2698 		device_printf(dev,
2699 		    "%d: type=%d seid=%d uplink=%d downlink=%d\n", i,
2700 		    sw_config->element[i].element_type,
2701 		    sw_config->element[i].seid,
2702 		    sw_config->element[i].uplink_seid,
2703 		    sw_config->element[i].downlink_seid);
2704 	}
2705 #endif
2706 	/* Simplified due to a single VSI at the moment */
2707 	vsi->uplink_seid = sw_config->element[0].uplink_seid;
2708 	vsi->downlink_seid = sw_config->element[0].downlink_seid;
2709 	vsi->seid = sw_config->element[0].seid;
2710 	return (ret);
2711 }
2712 
2713 /*********************************************************************
2714  *
2715  *  Initialize the VSI:  this handles contexts, which means things
2716  *  			 like the number of descriptors, buffer size,
2717  *			 plus we init the rings thru this function.
2718  *
2719  **********************************************************************/
2720 static int
2721 ixl_initialize_vsi(struct ixl_vsi *vsi)
2722 {
2723 	struct ixl_pf		*pf = vsi->back;
2724 	struct ixl_queue	*que = vsi->queues;
2725 	device_t		dev = vsi->dev;
2726 	struct i40e_hw		*hw = vsi->hw;
2727 	struct i40e_vsi_context	ctxt;
2728 	int			err = 0;
2729 
2730 	memset(&ctxt, 0, sizeof(ctxt));
2731 	ctxt.seid = vsi->seid;
2732 	if (pf->veb_seid != 0)
2733 		ctxt.uplink_seid = pf->veb_seid;
2734 	ctxt.pf_num = hw->pf_id;
2735 	err = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
2736 	if (err) {
2737 		device_printf(dev,"get vsi params failed %x!!\n", err);
2738 		return (err);
2739 	}
2740 #ifdef IXL_DEBUG
2741 	printf("get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, "
2742 	    "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, "
2743 	    "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid,
2744 	    ctxt.uplink_seid, ctxt.vsi_number,
2745 	    ctxt.vsis_allocated, ctxt.vsis_unallocated,
2746 	    ctxt.flags, ctxt.pf_num, ctxt.vf_num,
2747 	    ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits);
2748 #endif
2749 	/*
2750 	** Set the queue and traffic class bits
2751 	**  - when multiple traffic classes are supported
2752 	**    this will need to be more robust.
2753 	*/
2754 	ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
2755 	ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG;
2756 	ctxt.info.queue_mapping[0] = 0;
2757 	ctxt.info.tc_mapping[0] = 0x0800;
2758 
2759 	/* Set VLAN receive stripping mode */
2760 	ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID;
2761 	ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL;
2762 	if (vsi->ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
2763 	    ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2764 	else
2765 	    ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2766 
2767 	/* Keep copy of VSI info in VSI for statistic counters */
2768 	memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
2769 
2770 	/* Reset VSI statistics */
2771 	ixl_vsi_reset_stats(vsi);
2772 	vsi->hw_filters_add = 0;
2773 	vsi->hw_filters_del = 0;
2774 
2775 	ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF);
2776 
2777 	err = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
2778 	if (err) {
2779 		device_printf(dev,"update vsi params failed %x!!\n",
2780 		   hw->aq.asq_last_status);
2781 		return (err);
2782 	}
2783 
2784 	for (int i = 0; i < vsi->num_queues; i++, que++) {
2785 		struct tx_ring		*txr = &que->txr;
2786 		struct rx_ring 		*rxr = &que->rxr;
2787 		struct i40e_hmc_obj_txq tctx;
2788 		struct i40e_hmc_obj_rxq rctx;
2789 		u32			txctl;
2790 		u16			size;
2791 
2792 
2793 		/* Setup the HMC TX Context  */
2794 		size = que->num_desc * sizeof(struct i40e_tx_desc);
2795 		memset(&tctx, 0, sizeof(struct i40e_hmc_obj_txq));
2796 		tctx.new_context = 1;
2797 		tctx.base = (txr->dma.pa/IXL_TX_CTX_BASE_UNITS);
2798 		tctx.qlen = que->num_desc;
2799 		tctx.fc_ena = 0;
2800 		tctx.rdylist = vsi->info.qs_handle[0]; /* index is TC */
2801 		/* Enable HEAD writeback */
2802 		tctx.head_wb_ena = 1;
2803 		tctx.head_wb_addr = txr->dma.pa +
2804 		    (que->num_desc * sizeof(struct i40e_tx_desc));
2805 		tctx.rdylist_act = 0;
2806 		err = i40e_clear_lan_tx_queue_context(hw, i);
2807 		if (err) {
2808 			device_printf(dev, "Unable to clear TX context\n");
2809 			break;
2810 		}
2811 		err = i40e_set_lan_tx_queue_context(hw, i, &tctx);
2812 		if (err) {
2813 			device_printf(dev, "Unable to set TX context\n");
2814 			break;
2815 		}
2816 		/* Associate the ring with this PF */
2817 		txctl = I40E_QTX_CTL_PF_QUEUE;
2818 		txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
2819 		    I40E_QTX_CTL_PF_INDX_MASK);
2820 		wr32(hw, I40E_QTX_CTL(i), txctl);
2821 		ixl_flush(hw);
2822 
2823 		/* Do ring (re)init */
2824 		ixl_init_tx_ring(que);
2825 
2826 		/* Next setup the HMC RX Context  */
2827 		if (vsi->max_frame_size <= MCLBYTES)
2828 			rxr->mbuf_sz = MCLBYTES;
2829 		else
2830 			rxr->mbuf_sz = MJUMPAGESIZE;
2831 
2832 		u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len;
2833 
2834 		/* Set up an RX context for the HMC */
2835 		memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq));
2836 		rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT;
2837 		/* ignore header split for now */
2838 		rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT;
2839 		rctx.rxmax = (vsi->max_frame_size < max_rxmax) ?
2840 		    vsi->max_frame_size : max_rxmax;
2841 		rctx.dtype = 0;
2842 		rctx.dsize = 1;	/* do 32byte descriptors */
2843 		rctx.hsplit_0 = 0;  /* no HDR split initially */
2844 		rctx.base = (rxr->dma.pa/IXL_RX_CTX_BASE_UNITS);
2845 		rctx.qlen = que->num_desc;
2846 		rctx.tphrdesc_ena = 1;
2847 		rctx.tphwdesc_ena = 1;
2848 		rctx.tphdata_ena = 0;
2849 		rctx.tphhead_ena = 0;
2850 		rctx.lrxqthresh = 2;
2851 		rctx.crcstrip = 1;
2852 		rctx.l2tsel = 1;
2853 		rctx.showiv = 1;
2854 		rctx.fc_ena = 0;
2855 		rctx.prefena = 1;
2856 
2857 		err = i40e_clear_lan_rx_queue_context(hw, i);
2858 		if (err) {
2859 			device_printf(dev,
2860 			    "Unable to clear RX context %d\n", i);
2861 			break;
2862 		}
2863 		err = i40e_set_lan_rx_queue_context(hw, i, &rctx);
2864 		if (err) {
2865 			device_printf(dev, "Unable to set RX context %d\n", i);
2866 			break;
2867 		}
2868 		err = ixl_init_rx_ring(que);
2869 		if (err) {
2870 			device_printf(dev, "Fail in init_rx_ring %d\n", i);
2871 			break;
2872 		}
2873 		wr32(vsi->hw, I40E_QRX_TAIL(que->me), 0);
2874 #ifdef DEV_NETMAP
2875 		/* preserve queue */
2876 		if (vsi->ifp->if_capenable & IFCAP_NETMAP) {
2877 			struct netmap_adapter *na = NA(vsi->ifp);
2878 			struct netmap_kring *kring = &na->rx_rings[i];
2879 			int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
2880 			wr32(vsi->hw, I40E_QRX_TAIL(que->me), t);
2881 		} else
2882 #endif /* DEV_NETMAP */
2883 		wr32(vsi->hw, I40E_QRX_TAIL(que->me), que->num_desc - 1);
2884 	}
2885 	return (err);
2886 }
2887 
2888 
2889 /*********************************************************************
2890  *
2891  *  Free all VSI structs.
2892  *
2893  **********************************************************************/
2894 void
2895 ixl_free_vsi(struct ixl_vsi *vsi)
2896 {
2897 	struct ixl_pf		*pf = (struct ixl_pf *)vsi->back;
2898 	struct ixl_queue	*que = vsi->queues;
2899 
2900 	/* Free station queues */
2901 	for (int i = 0; i < vsi->num_queues; i++, que++) {
2902 		struct tx_ring *txr = &que->txr;
2903 		struct rx_ring *rxr = &que->rxr;
2904 
2905 		if (!mtx_initialized(&txr->mtx)) /* uninitialized */
2906 			continue;
2907 		IXL_TX_LOCK(txr);
2908 		ixl_free_que_tx(que);
2909 		if (txr->base)
2910 			i40e_free_dma_mem(&pf->hw, &txr->dma);
2911 		IXL_TX_UNLOCK(txr);
2912 		IXL_TX_LOCK_DESTROY(txr);
2913 
2914 		if (!mtx_initialized(&rxr->mtx)) /* uninitialized */
2915 			continue;
2916 		IXL_RX_LOCK(rxr);
2917 		ixl_free_que_rx(que);
2918 		if (rxr->base)
2919 			i40e_free_dma_mem(&pf->hw, &rxr->dma);
2920 		IXL_RX_UNLOCK(rxr);
2921 		IXL_RX_LOCK_DESTROY(rxr);
2922 
2923 	}
2924 	free(vsi->queues, M_DEVBUF);
2925 
2926 	/* Free VSI filter list */
2927 	ixl_free_mac_filters(vsi);
2928 }
2929 
2930 static void
2931 ixl_free_mac_filters(struct ixl_vsi *vsi)
2932 {
2933 	struct ixl_mac_filter *f;
2934 
2935 	while (!SLIST_EMPTY(&vsi->ftl)) {
2936 		f = SLIST_FIRST(&vsi->ftl);
2937 		SLIST_REMOVE_HEAD(&vsi->ftl, next);
2938 		free(f, M_DEVBUF);
2939 	}
2940 }
2941 
2942 
2943 /*********************************************************************
2944  *
2945  *  Allocate memory for the VSI (virtual station interface) and their
2946  *  associated queues, rings and the descriptors associated with each,
2947  *  called only once at attach.
2948  *
2949  **********************************************************************/
2950 static int
2951 ixl_setup_stations(struct ixl_pf *pf)
2952 {
2953 	device_t		dev = pf->dev;
2954 	struct ixl_vsi		*vsi;
2955 	struct ixl_queue	*que;
2956 	struct tx_ring		*txr;
2957 	struct rx_ring		*rxr;
2958 	int 			rsize, tsize;
2959 	int			error = I40E_SUCCESS;
2960 
2961 	vsi = &pf->vsi;
2962 	vsi->back = (void *)pf;
2963 	vsi->hw = &pf->hw;
2964 	vsi->id = 0;
2965 	vsi->num_vlans = 0;
2966 	vsi->back = pf;
2967 
2968 	/* Get memory for the station queues */
2969         if (!(vsi->queues =
2970             (struct ixl_queue *) malloc(sizeof(struct ixl_queue) *
2971             vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2972                 device_printf(dev, "Unable to allocate queue memory\n");
2973                 error = ENOMEM;
2974                 goto early;
2975         }
2976 
2977 	for (int i = 0; i < vsi->num_queues; i++) {
2978 		que = &vsi->queues[i];
2979 		que->num_desc = ixl_ringsz;
2980 		que->me = i;
2981 		que->vsi = vsi;
2982 		/* mark the queue as active */
2983 		vsi->active_queues |= (u64)1 << que->me;
2984 		txr = &que->txr;
2985 		txr->que = que;
2986 		txr->tail = I40E_QTX_TAIL(que->me);
2987 
2988 		/* Initialize the TX lock */
2989 		snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2990 		    device_get_nameunit(dev), que->me);
2991 		mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF);
2992 		/* Create the TX descriptor ring */
2993 		tsize = roundup2((que->num_desc *
2994 		    sizeof(struct i40e_tx_desc)) +
2995 		    sizeof(u32), DBA_ALIGN);
2996 		if (i40e_allocate_dma_mem(&pf->hw,
2997 		    &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) {
2998 			device_printf(dev,
2999 			    "Unable to allocate TX Descriptor memory\n");
3000 			error = ENOMEM;
3001 			goto fail;
3002 		}
3003 		txr->base = (struct i40e_tx_desc *)txr->dma.va;
3004 		bzero((void *)txr->base, tsize);
3005        		/* Now allocate transmit soft structs for the ring */
3006        		if (ixl_allocate_tx_data(que)) {
3007 			device_printf(dev,
3008 			    "Critical Failure setting up TX structures\n");
3009 			error = ENOMEM;
3010 			goto fail;
3011        		}
3012 		/* Allocate a buf ring */
3013 		txr->br = buf_ring_alloc(4096, M_DEVBUF,
3014 		    M_WAITOK, &txr->mtx);
3015 		if (txr->br == NULL) {
3016 			device_printf(dev,
3017 			    "Critical Failure setting up TX buf ring\n");
3018 			error = ENOMEM;
3019 			goto fail;
3020        		}
3021 
3022 		/*
3023 		 * Next the RX queues...
3024 		 */
3025 		rsize = roundup2(que->num_desc *
3026 		    sizeof(union i40e_rx_desc), DBA_ALIGN);
3027 		rxr = &que->rxr;
3028 		rxr->que = que;
3029 		rxr->tail = I40E_QRX_TAIL(que->me);
3030 
3031 		/* Initialize the RX side lock */
3032 		snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3033 		    device_get_nameunit(dev), que->me);
3034 		mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF);
3035 
3036 		if (i40e_allocate_dma_mem(&pf->hw,
3037 		    &rxr->dma, i40e_mem_reserved, rsize, 4096)) {
3038 			device_printf(dev,
3039 			    "Unable to allocate RX Descriptor memory\n");
3040 			error = ENOMEM;
3041 			goto fail;
3042 		}
3043 		rxr->base = (union i40e_rx_desc *)rxr->dma.va;
3044 		bzero((void *)rxr->base, rsize);
3045 
3046         	/* Allocate receive soft structs for the ring*/
3047 		if (ixl_allocate_rx_data(que)) {
3048 			device_printf(dev,
3049 			    "Critical Failure setting up receive structs\n");
3050 			error = ENOMEM;
3051 			goto fail;
3052 		}
3053 	}
3054 
3055 	return (0);
3056 
3057 fail:
3058 	for (int i = 0; i < vsi->num_queues; i++) {
3059 		que = &vsi->queues[i];
3060 		rxr = &que->rxr;
3061 		txr = &que->txr;
3062 		if (rxr->base)
3063 			i40e_free_dma_mem(&pf->hw, &rxr->dma);
3064 		if (txr->base)
3065 			i40e_free_dma_mem(&pf->hw, &txr->dma);
3066 	}
3067 
3068 early:
3069 	return (error);
3070 }
3071 
3072 /*
3073 ** Provide a update to the queue RX
3074 ** interrupt moderation value.
3075 */
3076 static void
3077 ixl_set_queue_rx_itr(struct ixl_queue *que)
3078 {
3079 	struct ixl_vsi	*vsi = que->vsi;
3080 	struct i40e_hw	*hw = vsi->hw;
3081 	struct rx_ring	*rxr = &que->rxr;
3082 	u16		rx_itr;
3083 	u16		rx_latency = 0;
3084 	int		rx_bytes;
3085 
3086 
3087 	/* Idle, do nothing */
3088 	if (rxr->bytes == 0)
3089 		return;
3090 
3091 	if (ixl_dynamic_rx_itr) {
3092 		rx_bytes = rxr->bytes/rxr->itr;
3093 		rx_itr = rxr->itr;
3094 
3095 		/* Adjust latency range */
3096 		switch (rxr->latency) {
3097 		case IXL_LOW_LATENCY:
3098 			if (rx_bytes > 10) {
3099 				rx_latency = IXL_AVE_LATENCY;
3100 				rx_itr = IXL_ITR_20K;
3101 			}
3102 			break;
3103 		case IXL_AVE_LATENCY:
3104 			if (rx_bytes > 20) {
3105 				rx_latency = IXL_BULK_LATENCY;
3106 				rx_itr = IXL_ITR_8K;
3107 			} else if (rx_bytes <= 10) {
3108 				rx_latency = IXL_LOW_LATENCY;
3109 				rx_itr = IXL_ITR_100K;
3110 			}
3111 			break;
3112 		case IXL_BULK_LATENCY:
3113 			if (rx_bytes <= 20) {
3114 				rx_latency = IXL_AVE_LATENCY;
3115 				rx_itr = IXL_ITR_20K;
3116 			}
3117 			break;
3118        		 }
3119 
3120 		rxr->latency = rx_latency;
3121 
3122 		if (rx_itr != rxr->itr) {
3123 			/* do an exponential smoothing */
3124 			rx_itr = (10 * rx_itr * rxr->itr) /
3125 			    ((9 * rx_itr) + rxr->itr);
3126 			rxr->itr = rx_itr & IXL_MAX_ITR;
3127 			wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3128 			    que->me), rxr->itr);
3129 		}
3130 	} else { /* We may have have toggled to non-dynamic */
3131 		if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC)
3132 			vsi->rx_itr_setting = ixl_rx_itr;
3133 		/* Update the hardware if needed */
3134 		if (rxr->itr != vsi->rx_itr_setting) {
3135 			rxr->itr = vsi->rx_itr_setting;
3136 			wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3137 			    que->me), rxr->itr);
3138 		}
3139 	}
3140 	rxr->bytes = 0;
3141 	rxr->packets = 0;
3142 	return;
3143 }
3144 
3145 
3146 /*
3147 ** Provide a update to the queue TX
3148 ** interrupt moderation value.
3149 */
3150 static void
3151 ixl_set_queue_tx_itr(struct ixl_queue *que)
3152 {
3153 	struct ixl_vsi	*vsi = que->vsi;
3154 	struct i40e_hw	*hw = vsi->hw;
3155 	struct tx_ring	*txr = &que->txr;
3156 	u16		tx_itr;
3157 	u16		tx_latency = 0;
3158 	int		tx_bytes;
3159 
3160 
3161 	/* Idle, do nothing */
3162 	if (txr->bytes == 0)
3163 		return;
3164 
3165 	if (ixl_dynamic_tx_itr) {
3166 		tx_bytes = txr->bytes/txr->itr;
3167 		tx_itr = txr->itr;
3168 
3169 		switch (txr->latency) {
3170 		case IXL_LOW_LATENCY:
3171 			if (tx_bytes > 10) {
3172 				tx_latency = IXL_AVE_LATENCY;
3173 				tx_itr = IXL_ITR_20K;
3174 			}
3175 			break;
3176 		case IXL_AVE_LATENCY:
3177 			if (tx_bytes > 20) {
3178 				tx_latency = IXL_BULK_LATENCY;
3179 				tx_itr = IXL_ITR_8K;
3180 			} else if (tx_bytes <= 10) {
3181 				tx_latency = IXL_LOW_LATENCY;
3182 				tx_itr = IXL_ITR_100K;
3183 			}
3184 			break;
3185 		case IXL_BULK_LATENCY:
3186 			if (tx_bytes <= 20) {
3187 				tx_latency = IXL_AVE_LATENCY;
3188 				tx_itr = IXL_ITR_20K;
3189 			}
3190 			break;
3191 		}
3192 
3193 		txr->latency = tx_latency;
3194 
3195 		if (tx_itr != txr->itr) {
3196        	         /* do an exponential smoothing */
3197 			tx_itr = (10 * tx_itr * txr->itr) /
3198 			    ((9 * tx_itr) + txr->itr);
3199 			txr->itr = tx_itr & IXL_MAX_ITR;
3200 			wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3201 			    que->me), txr->itr);
3202 		}
3203 
3204 	} else { /* We may have have toggled to non-dynamic */
3205 		if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC)
3206 			vsi->tx_itr_setting = ixl_tx_itr;
3207 		/* Update the hardware if needed */
3208 		if (txr->itr != vsi->tx_itr_setting) {
3209 			txr->itr = vsi->tx_itr_setting;
3210 			wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3211 			    que->me), txr->itr);
3212 		}
3213 	}
3214 	txr->bytes = 0;
3215 	txr->packets = 0;
3216 	return;
3217 }
3218 
3219 #define QUEUE_NAME_LEN 32
3220 
3221 static void
3222 ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi,
3223     struct sysctl_ctx_list *ctx, const char *sysctl_name)
3224 {
3225 	struct sysctl_oid *tree;
3226 	struct sysctl_oid_list *child;
3227 	struct sysctl_oid_list *vsi_list;
3228 
3229 	tree = device_get_sysctl_tree(pf->dev);
3230 	child = SYSCTL_CHILDREN(tree);
3231 	vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name,
3232 				   CTLFLAG_RD, NULL, "VSI Number");
3233 	vsi_list = SYSCTL_CHILDREN(vsi->vsi_node);
3234 
3235 	ixl_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats);
3236 }
3237 
3238 static void
3239 ixl_add_hw_stats(struct ixl_pf *pf)
3240 {
3241 	device_t dev = pf->dev;
3242 	struct ixl_vsi *vsi = &pf->vsi;
3243 	struct ixl_queue *queues = vsi->queues;
3244 	struct i40e_hw_port_stats *pf_stats = &pf->stats;
3245 
3246 	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
3247 	struct sysctl_oid *tree = device_get_sysctl_tree(dev);
3248 	struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
3249 	struct sysctl_oid_list *vsi_list;
3250 
3251 	struct sysctl_oid *queue_node;
3252 	struct sysctl_oid_list *queue_list;
3253 
3254 	struct tx_ring *txr;
3255 	struct rx_ring *rxr;
3256 	char queue_namebuf[QUEUE_NAME_LEN];
3257 
3258 	/* Driver statistics */
3259 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
3260 			CTLFLAG_RD, &pf->watchdog_events,
3261 			"Watchdog timeouts");
3262 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq",
3263 			CTLFLAG_RD, &pf->admin_irq,
3264 			"Admin Queue IRQ Handled");
3265 
3266 	ixl_add_vsi_sysctls(pf, &pf->vsi, ctx, "pf");
3267 	vsi_list = SYSCTL_CHILDREN(pf->vsi.vsi_node);
3268 
3269 	/* Queue statistics */
3270 	for (int q = 0; q < vsi->num_queues; q++) {
3271 		snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q);
3272 		queue_node = SYSCTL_ADD_NODE(ctx, vsi_list,
3273 		    OID_AUTO, queue_namebuf, CTLFLAG_RD, NULL, "Queue #");
3274 		queue_list = SYSCTL_CHILDREN(queue_node);
3275 
3276 		txr = &(queues[q].txr);
3277 		rxr = &(queues[q].rxr);
3278 
3279 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed",
3280 				CTLFLAG_RD, &(queues[q].mbuf_defrag_failed),
3281 				"m_defrag() failed");
3282 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "dropped",
3283 				CTLFLAG_RD, &(queues[q].dropped_pkts),
3284 				"Driver dropped packets");
3285 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
3286 				CTLFLAG_RD, &(queues[q].irqs),
3287 				"irqs on this queue");
3288 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx",
3289 				CTLFLAG_RD, &(queues[q].tso),
3290 				"TSO");
3291 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_dma_setup",
3292 				CTLFLAG_RD, &(queues[q].tx_dma_setup),
3293 				"Driver tx dma failure in xmit");
3294 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
3295 				CTLFLAG_RD, &(txr->no_desc),
3296 				"Queue No Descriptor Available");
3297 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
3298 				CTLFLAG_RD, &(txr->total_packets),
3299 				"Queue Packets Transmitted");
3300 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_bytes",
3301 				CTLFLAG_RD, &(txr->tx_bytes),
3302 				"Queue Bytes Transmitted");
3303 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
3304 				CTLFLAG_RD, &(rxr->rx_packets),
3305 				"Queue Packets Received");
3306 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
3307 				CTLFLAG_RD, &(rxr->rx_bytes),
3308 				"Queue Bytes Received");
3309 	}
3310 
3311 	/* MAC stats */
3312 	ixl_add_sysctls_mac_stats(ctx, child, pf_stats);
3313 }
3314 
3315 static void
3316 ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx,
3317 	struct sysctl_oid_list *child,
3318 	struct i40e_eth_stats *eth_stats)
3319 {
3320 	struct ixl_sysctl_info ctls[] =
3321 	{
3322 		{&eth_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"},
3323 		{&eth_stats->rx_unicast, "ucast_pkts_rcvd",
3324 			"Unicast Packets Received"},
3325 		{&eth_stats->rx_multicast, "mcast_pkts_rcvd",
3326 			"Multicast Packets Received"},
3327 		{&eth_stats->rx_broadcast, "bcast_pkts_rcvd",
3328 			"Broadcast Packets Received"},
3329 		{&eth_stats->rx_discards, "rx_discards", "Discarded RX packets"},
3330 		{&eth_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"},
3331 		{&eth_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"},
3332 		{&eth_stats->tx_multicast, "mcast_pkts_txd",
3333 			"Multicast Packets Transmitted"},
3334 		{&eth_stats->tx_broadcast, "bcast_pkts_txd",
3335 			"Broadcast Packets Transmitted"},
3336 		// end
3337 		{0,0,0}
3338 	};
3339 
3340 	struct ixl_sysctl_info *entry = ctls;
3341 	while (entry->stat != 0)
3342 	{
3343 		SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name,
3344 				CTLFLAG_RD, entry->stat,
3345 				entry->description);
3346 		entry++;
3347 	}
3348 }
3349 
3350 static void
3351 ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *ctx,
3352 	struct sysctl_oid_list *child,
3353 	struct i40e_hw_port_stats *stats)
3354 {
3355 	struct sysctl_oid *stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac",
3356 				    CTLFLAG_RD, NULL, "Mac Statistics");
3357 	struct sysctl_oid_list *stat_list = SYSCTL_CHILDREN(stat_node);
3358 
3359 	struct i40e_eth_stats *eth_stats = &stats->eth;
3360 	ixl_add_sysctls_eth_stats(ctx, stat_list, eth_stats);
3361 
3362 	struct ixl_sysctl_info ctls[] =
3363 	{
3364 		{&stats->crc_errors, "crc_errors", "CRC Errors"},
3365 		{&stats->illegal_bytes, "illegal_bytes", "Illegal Byte Errors"},
3366 		{&stats->mac_local_faults, "local_faults", "MAC Local Faults"},
3367 		{&stats->mac_remote_faults, "remote_faults", "MAC Remote Faults"},
3368 		{&stats->rx_length_errors, "rx_length_errors", "Receive Length Errors"},
3369 		/* Packet Reception Stats */
3370 		{&stats->rx_size_64, "rx_frames_64", "64 byte frames received"},
3371 		{&stats->rx_size_127, "rx_frames_65_127", "65-127 byte frames received"},
3372 		{&stats->rx_size_255, "rx_frames_128_255", "128-255 byte frames received"},
3373 		{&stats->rx_size_511, "rx_frames_256_511", "256-511 byte frames received"},
3374 		{&stats->rx_size_1023, "rx_frames_512_1023", "512-1023 byte frames received"},
3375 		{&stats->rx_size_1522, "rx_frames_1024_1522", "1024-1522 byte frames received"},
3376 		{&stats->rx_size_big, "rx_frames_big", "1523-9522 byte frames received"},
3377 		{&stats->rx_undersize, "rx_undersize", "Undersized packets received"},
3378 		{&stats->rx_fragments, "rx_fragmented", "Fragmented packets received"},
3379 		{&stats->rx_oversize, "rx_oversized", "Oversized packets received"},
3380 		{&stats->rx_jabber, "rx_jabber", "Received Jabber"},
3381 		{&stats->checksum_error, "checksum_errors", "Checksum Errors"},
3382 		/* Packet Transmission Stats */
3383 		{&stats->tx_size_64, "tx_frames_64", "64 byte frames transmitted"},
3384 		{&stats->tx_size_127, "tx_frames_65_127", "65-127 byte frames transmitted"},
3385 		{&stats->tx_size_255, "tx_frames_128_255", "128-255 byte frames transmitted"},
3386 		{&stats->tx_size_511, "tx_frames_256_511", "256-511 byte frames transmitted"},
3387 		{&stats->tx_size_1023, "tx_frames_512_1023", "512-1023 byte frames transmitted"},
3388 		{&stats->tx_size_1522, "tx_frames_1024_1522", "1024-1522 byte frames transmitted"},
3389 		{&stats->tx_size_big, "tx_frames_big", "1523-9522 byte frames transmitted"},
3390 		/* Flow control */
3391 		{&stats->link_xon_tx, "xon_txd", "Link XON transmitted"},
3392 		{&stats->link_xon_rx, "xon_recvd", "Link XON received"},
3393 		{&stats->link_xoff_tx, "xoff_txd", "Link XOFF transmitted"},
3394 		{&stats->link_xoff_rx, "xoff_recvd", "Link XOFF received"},
3395 		/* End */
3396 		{0,0,0}
3397 	};
3398 
3399 	struct ixl_sysctl_info *entry = ctls;
3400 	while (entry->stat != 0)
3401 	{
3402 		SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, entry->name,
3403 				CTLFLAG_RD, entry->stat,
3404 				entry->description);
3405 		entry++;
3406 	}
3407 }
3408 
3409 
3410 /*
3411 ** ixl_config_rss - setup RSS
3412 **  - note this is done for the single vsi
3413 */
3414 static void ixl_config_rss(struct ixl_vsi *vsi)
3415 {
3416 	struct ixl_pf	*pf = (struct ixl_pf *)vsi->back;
3417 	struct i40e_hw	*hw = vsi->hw;
3418 	u32		lut = 0;
3419 	u64		set_hena = 0, hena;
3420 	int		i, j, que_id;
3421 #ifdef RSS
3422 	u32		rss_hash_config;
3423 	u32		rss_seed[IXL_KEYSZ];
3424 #else
3425 	u32             rss_seed[IXL_KEYSZ] = {0x41b01687,
3426 			    0x183cfd8c, 0xce880440, 0x580cbc3c,
3427 			    0x35897377, 0x328b25e1, 0x4fa98922,
3428 			    0xb7d90c14, 0xd5bad70d, 0xcd15a2c1};
3429 #endif
3430 
3431 #ifdef RSS
3432         /* Fetch the configured RSS key */
3433         rss_getkey((uint8_t *) &rss_seed);
3434 #endif
3435 
3436 	/* Fill out hash function seed */
3437 	for (i = 0; i < IXL_KEYSZ; i++)
3438                 wr32(hw, I40E_PFQF_HKEY(i), rss_seed[i]);
3439 
3440 	/* Enable PCTYPES for RSS: */
3441 #ifdef RSS
3442 	rss_hash_config = rss_gethashconfig();
3443 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
3444                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER);
3445 	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
3446                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP);
3447 	if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
3448                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP);
3449 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
3450                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER);
3451 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
3452 		set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6);
3453 	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
3454                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP);
3455         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
3456                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP);
3457 #else
3458 	set_hena =
3459 		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) |
3460 		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) |
3461 		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) |
3462 		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) |
3463 		((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) |
3464 		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) |
3465 		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) |
3466 		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) |
3467 		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) |
3468 		((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) |
3469 		((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD);
3470 #endif
3471 	hena = (u64)rd32(hw, I40E_PFQF_HENA(0)) |
3472 	    ((u64)rd32(hw, I40E_PFQF_HENA(1)) << 32);
3473 	hena |= set_hena;
3474 	wr32(hw, I40E_PFQF_HENA(0), (u32)hena);
3475 	wr32(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
3476 
3477 	/* Populate the LUT with max no. of queues in round robin fashion */
3478 	for (i = j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) {
3479 		if (j == vsi->num_queues)
3480 			j = 0;
3481 #ifdef RSS
3482 		/*
3483 		 * Fetch the RSS bucket id for the given indirection entry.
3484 		 * Cap it at the number of configured buckets (which is
3485 		 * num_queues.)
3486 		 */
3487 		que_id = rss_get_indirection_to_bucket(i);
3488 		que_id = que_id % vsi->num_queues;
3489 #else
3490 		que_id = j;
3491 #endif
3492 		/* lut = 4-byte sliding window of 4 lut entries */
3493 		lut = (lut << 8) | (que_id &
3494 		    ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1));
3495 		/* On i = 3, we have 4 entries in lut; write to the register */
3496 		if ((i & 3) == 3)
3497 			wr32(hw, I40E_PFQF_HLUT(i >> 2), lut);
3498 	}
3499 	ixl_flush(hw);
3500 }
3501 
3502 
3503 /*
3504 ** This routine is run via an vlan config EVENT,
3505 ** it enables us to use the HW Filter table since
3506 ** we can get the vlan id. This just creates the
3507 ** entry in the soft version of the VFTA, init will
3508 ** repopulate the real table.
3509 */
3510 static void
3511 ixl_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3512 {
3513 	struct ixl_vsi	*vsi = ifp->if_softc;
3514 	struct i40e_hw	*hw = vsi->hw;
3515 	struct ixl_pf	*pf = (struct ixl_pf *)vsi->back;
3516 
3517 	if (ifp->if_softc !=  arg)   /* Not our event */
3518 		return;
3519 
3520 	if ((vtag == 0) || (vtag > 4095))	/* Invalid */
3521 		return;
3522 
3523 	IXL_PF_LOCK(pf);
3524 	++vsi->num_vlans;
3525 	ixl_add_filter(vsi, hw->mac.addr, vtag);
3526 	IXL_PF_UNLOCK(pf);
3527 }
3528 
3529 /*
3530 ** This routine is run via an vlan
3531 ** unconfig EVENT, remove our entry
3532 ** in the soft vfta.
3533 */
3534 static void
3535 ixl_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3536 {
3537 	struct ixl_vsi	*vsi = ifp->if_softc;
3538 	struct i40e_hw	*hw = vsi->hw;
3539 	struct ixl_pf	*pf = (struct ixl_pf *)vsi->back;
3540 
3541 	if (ifp->if_softc !=  arg)
3542 		return;
3543 
3544 	if ((vtag == 0) || (vtag > 4095))	/* Invalid */
3545 		return;
3546 
3547 	IXL_PF_LOCK(pf);
3548 	--vsi->num_vlans;
3549 	ixl_del_filter(vsi, hw->mac.addr, vtag);
3550 	IXL_PF_UNLOCK(pf);
3551 }
3552 
3553 /*
3554 ** This routine updates vlan filters, called by init
3555 ** it scans the filter table and then updates the hw
3556 ** after a soft reset.
3557 */
3558 static void
3559 ixl_setup_vlan_filters(struct ixl_vsi *vsi)
3560 {
3561 	struct ixl_mac_filter	*f;
3562 	int			cnt = 0, flags;
3563 
3564 	if (vsi->num_vlans == 0)
3565 		return;
3566 	/*
3567 	** Scan the filter list for vlan entries,
3568 	** mark them for addition and then call
3569 	** for the AQ update.
3570 	*/
3571 	SLIST_FOREACH(f, &vsi->ftl, next) {
3572 		if (f->flags & IXL_FILTER_VLAN) {
3573 			f->flags |=
3574 			    (IXL_FILTER_ADD |
3575 			    IXL_FILTER_USED);
3576 			cnt++;
3577 		}
3578 	}
3579 	if (cnt == 0) {
3580 		printf("setup vlan: no filters found!\n");
3581 		return;
3582 	}
3583 	flags = IXL_FILTER_VLAN;
3584 	flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
3585 	ixl_add_hw_filters(vsi, flags, cnt);
3586 	return;
3587 }
3588 
3589 /*
3590 ** Initialize filter list and add filters that the hardware
3591 ** needs to know about.
3592 */
3593 static void
3594 ixl_init_filters(struct ixl_vsi *vsi)
3595 {
3596 	/* Add broadcast address */
3597 	ixl_add_filter(vsi, ixl_bcast_addr, IXL_VLAN_ANY);
3598 }
3599 
3600 /*
3601 ** This routine adds mulicast filters
3602 */
3603 static void
3604 ixl_add_mc_filter(struct ixl_vsi *vsi, u8 *macaddr)
3605 {
3606 	struct ixl_mac_filter *f;
3607 
3608 	/* Does one already exist */
3609 	f = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
3610 	if (f != NULL)
3611 		return;
3612 
3613 	f = ixl_get_filter(vsi);
3614 	if (f == NULL) {
3615 		printf("WARNING: no filter available!!\n");
3616 		return;
3617 	}
3618 	bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
3619 	f->vlan = IXL_VLAN_ANY;
3620 	f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED
3621 	    | IXL_FILTER_MC);
3622 
3623 	return;
3624 }
3625 
3626 static void
3627 ixl_reconfigure_filters(struct ixl_vsi *vsi)
3628 {
3629 
3630 	ixl_add_hw_filters(vsi, IXL_FILTER_USED, vsi->num_macs);
3631 }
3632 
3633 /*
3634 ** This routine adds macvlan filters
3635 */
3636 static void
3637 ixl_add_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
3638 {
3639 	struct ixl_mac_filter	*f, *tmp;
3640 	struct ixl_pf		*pf;
3641 	device_t		dev;
3642 
3643 	DEBUGOUT("ixl_add_filter: begin");
3644 
3645 	pf = vsi->back;
3646 	dev = pf->dev;
3647 
3648 	/* Does one already exist */
3649 	f = ixl_find_filter(vsi, macaddr, vlan);
3650 	if (f != NULL)
3651 		return;
3652 	/*
3653 	** Is this the first vlan being registered, if so we
3654 	** need to remove the ANY filter that indicates we are
3655 	** not in a vlan, and replace that with a 0 filter.
3656 	*/
3657 	if ((vlan != IXL_VLAN_ANY) && (vsi->num_vlans == 1)) {
3658 		tmp = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
3659 		if (tmp != NULL) {
3660 			ixl_del_filter(vsi, macaddr, IXL_VLAN_ANY);
3661 			ixl_add_filter(vsi, macaddr, 0);
3662 		}
3663 	}
3664 
3665 	f = ixl_get_filter(vsi);
3666 	if (f == NULL) {
3667 		device_printf(dev, "WARNING: no filter available!!\n");
3668 		return;
3669 	}
3670 	bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
3671 	f->vlan = vlan;
3672 	f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
3673 	if (f->vlan != IXL_VLAN_ANY)
3674 		f->flags |= IXL_FILTER_VLAN;
3675 	else
3676 		vsi->num_macs++;
3677 
3678 	ixl_add_hw_filters(vsi, f->flags, 1);
3679 	return;
3680 }
3681 
3682 static void
3683 ixl_del_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
3684 {
3685 	struct ixl_mac_filter *f;
3686 
3687 	f = ixl_find_filter(vsi, macaddr, vlan);
3688 	if (f == NULL)
3689 		return;
3690 
3691 	f->flags |= IXL_FILTER_DEL;
3692 	ixl_del_hw_filters(vsi, 1);
3693 	vsi->num_macs--;
3694 
3695 	/* Check if this is the last vlan removal */
3696 	if (vlan != IXL_VLAN_ANY && vsi->num_vlans == 0) {
3697 		/* Switch back to a non-vlan filter */
3698 		ixl_del_filter(vsi, macaddr, 0);
3699 		ixl_add_filter(vsi, macaddr, IXL_VLAN_ANY);
3700 	}
3701 	return;
3702 }
3703 
3704 /*
3705 ** Find the filter with both matching mac addr and vlan id
3706 */
3707 static struct ixl_mac_filter *
3708 ixl_find_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
3709 {
3710 	struct ixl_mac_filter	*f;
3711 	bool			match = FALSE;
3712 
3713 	SLIST_FOREACH(f, &vsi->ftl, next) {
3714 		if (!cmp_etheraddr(f->macaddr, macaddr))
3715 			continue;
3716 		if (f->vlan == vlan) {
3717 			match = TRUE;
3718 			break;
3719 		}
3720 	}
3721 
3722 	if (!match)
3723 		f = NULL;
3724 	return (f);
3725 }
3726 
3727 /*
3728 ** This routine takes additions to the vsi filter
3729 ** table and creates an Admin Queue call to create
3730 ** the filters in the hardware.
3731 */
3732 static void
3733 ixl_add_hw_filters(struct ixl_vsi *vsi, int flags, int cnt)
3734 {
3735 	struct i40e_aqc_add_macvlan_element_data *a, *b;
3736 	struct ixl_mac_filter	*f;
3737 	struct ixl_pf		*pf;
3738 	struct i40e_hw		*hw;
3739 	device_t		dev;
3740 	int			err, j = 0;
3741 
3742 	pf = vsi->back;
3743 	dev = pf->dev;
3744 	hw = &pf->hw;
3745 	IXL_PF_LOCK_ASSERT(pf);
3746 
3747 	a = malloc(sizeof(struct i40e_aqc_add_macvlan_element_data) * cnt,
3748 	    M_DEVBUF, M_NOWAIT | M_ZERO);
3749 	if (a == NULL) {
3750 		device_printf(dev, "add_hw_filters failed to get memory\n");
3751 		return;
3752 	}
3753 
3754 	/*
3755 	** Scan the filter list, each time we find one
3756 	** we add it to the admin queue array and turn off
3757 	** the add bit.
3758 	*/
3759 	SLIST_FOREACH(f, &vsi->ftl, next) {
3760 		if (f->flags == flags) {
3761 			b = &a[j]; // a pox on fvl long names :)
3762 			bcopy(f->macaddr, b->mac_addr, ETHER_ADDR_LEN);
3763 			if (f->vlan == IXL_VLAN_ANY) {
3764 				b->vlan_tag = 0;
3765 				b->flags = I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
3766 			} else {
3767 				b->vlan_tag = f->vlan;
3768 				b->flags = 0;
3769 			}
3770 			b->flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
3771 			f->flags &= ~IXL_FILTER_ADD;
3772 			j++;
3773 		}
3774 		if (j == cnt)
3775 			break;
3776 	}
3777 	if (j > 0) {
3778 		err = i40e_aq_add_macvlan(hw, vsi->seid, a, j, NULL);
3779 		if (err)
3780 			device_printf(dev, "aq_add_macvlan err %d, "
3781 			    "aq_error %d\n", err, hw->aq.asq_last_status);
3782 		else
3783 			vsi->hw_filters_add += j;
3784 	}
3785 	free(a, M_DEVBUF);
3786 	return;
3787 }
3788 
3789 /*
3790 ** This routine takes removals in the vsi filter
3791 ** table and creates an Admin Queue call to delete
3792 ** the filters in the hardware.
3793 */
3794 static void
3795 ixl_del_hw_filters(struct ixl_vsi *vsi, int cnt)
3796 {
3797 	struct i40e_aqc_remove_macvlan_element_data *d, *e;
3798 	struct ixl_pf		*pf;
3799 	struct i40e_hw		*hw;
3800 	device_t		dev;
3801 	struct ixl_mac_filter	*f, *f_temp;
3802 	int			err, j = 0;
3803 
3804 	DEBUGOUT("ixl_del_hw_filters: begin\n");
3805 
3806 	pf = vsi->back;
3807 	hw = &pf->hw;
3808 	dev = pf->dev;
3809 
3810 	d = malloc(sizeof(struct i40e_aqc_remove_macvlan_element_data) * cnt,
3811 	    M_DEVBUF, M_NOWAIT | M_ZERO);
3812 	if (d == NULL) {
3813 		printf("del hw filter failed to get memory\n");
3814 		return;
3815 	}
3816 
3817 	SLIST_FOREACH_SAFE(f, &vsi->ftl, next, f_temp) {
3818 		if (f->flags & IXL_FILTER_DEL) {
3819 			e = &d[j]; // a pox on fvl long names :)
3820 			bcopy(f->macaddr, e->mac_addr, ETHER_ADDR_LEN);
3821 			e->vlan_tag = (f->vlan == IXL_VLAN_ANY ? 0 : f->vlan);
3822 			e->flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
3823 			/* delete entry from vsi list */
3824 			SLIST_REMOVE(&vsi->ftl, f, ixl_mac_filter, next);
3825 			free(f, M_DEVBUF);
3826 			j++;
3827 		}
3828 		if (j == cnt)
3829 			break;
3830 	}
3831 	if (j > 0) {
3832 		err = i40e_aq_remove_macvlan(hw, vsi->seid, d, j, NULL);
3833 		/* NOTE: returns ENOENT every time but seems to work fine,
3834 		   so we'll ignore that specific error. */
3835 		// TODO: Does this still occur on current firmwares?
3836 		if (err && hw->aq.asq_last_status != I40E_AQ_RC_ENOENT) {
3837 			int sc = 0;
3838 			for (int i = 0; i < j; i++)
3839 				sc += (!d[i].error_code);
3840 			vsi->hw_filters_del += sc;
3841 			device_printf(dev,
3842 			    "Failed to remove %d/%d filters, aq error %d\n",
3843 			    j - sc, j, hw->aq.asq_last_status);
3844 		} else
3845 			vsi->hw_filters_del += j;
3846 	}
3847 	free(d, M_DEVBUF);
3848 
3849 	DEBUGOUT("ixl_del_hw_filters: end\n");
3850 	return;
3851 }
3852 
3853 static int
3854 ixl_enable_rings(struct ixl_vsi *vsi)
3855 {
3856 	struct ixl_pf	*pf = vsi->back;
3857 	struct i40e_hw	*hw = &pf->hw;
3858 	int		index, error;
3859 	u32		reg;
3860 
3861 	error = 0;
3862 	for (int i = 0; i < vsi->num_queues; i++) {
3863 		index = vsi->first_queue + i;
3864 		i40e_pre_tx_queue_cfg(hw, index, TRUE);
3865 
3866 		reg = rd32(hw, I40E_QTX_ENA(index));
3867 		reg |= I40E_QTX_ENA_QENA_REQ_MASK |
3868 		    I40E_QTX_ENA_QENA_STAT_MASK;
3869 		wr32(hw, I40E_QTX_ENA(index), reg);
3870 		/* Verify the enable took */
3871 		for (int j = 0; j < 10; j++) {
3872 			reg = rd32(hw, I40E_QTX_ENA(index));
3873 			if (reg & I40E_QTX_ENA_QENA_STAT_MASK)
3874 				break;
3875 			i40e_msec_delay(10);
3876 		}
3877 		if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) {
3878 			device_printf(pf->dev, "TX queue %d disabled!\n",
3879 			    index);
3880 			error = ETIMEDOUT;
3881 		}
3882 
3883 		reg = rd32(hw, I40E_QRX_ENA(index));
3884 		reg |= I40E_QRX_ENA_QENA_REQ_MASK |
3885 		    I40E_QRX_ENA_QENA_STAT_MASK;
3886 		wr32(hw, I40E_QRX_ENA(index), reg);
3887 		/* Verify the enable took */
3888 		for (int j = 0; j < 10; j++) {
3889 			reg = rd32(hw, I40E_QRX_ENA(index));
3890 			if (reg & I40E_QRX_ENA_QENA_STAT_MASK)
3891 				break;
3892 			i40e_msec_delay(10);
3893 		}
3894 		if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) {
3895 			device_printf(pf->dev, "RX queue %d disabled!\n",
3896 			    index);
3897 			error = ETIMEDOUT;
3898 		}
3899 	}
3900 
3901 	return (error);
3902 }
3903 
3904 static int
3905 ixl_disable_rings(struct ixl_vsi *vsi)
3906 {
3907 	struct ixl_pf	*pf = vsi->back;
3908 	struct i40e_hw	*hw = &pf->hw;
3909 	int		index, error;
3910 	u32		reg;
3911 
3912 	error = 0;
3913 	for (int i = 0; i < vsi->num_queues; i++) {
3914 		index = vsi->first_queue + i;
3915 
3916 		i40e_pre_tx_queue_cfg(hw, index, FALSE);
3917 		i40e_usec_delay(500);
3918 
3919 		reg = rd32(hw, I40E_QTX_ENA(index));
3920 		reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
3921 		wr32(hw, I40E_QTX_ENA(index), reg);
3922 		/* Verify the disable took */
3923 		for (int j = 0; j < 10; j++) {
3924 			reg = rd32(hw, I40E_QTX_ENA(index));
3925 			if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
3926 				break;
3927 			i40e_msec_delay(10);
3928 		}
3929 		if (reg & I40E_QTX_ENA_QENA_STAT_MASK) {
3930 			device_printf(pf->dev, "TX queue %d still enabled!\n",
3931 			    index);
3932 			error = ETIMEDOUT;
3933 		}
3934 
3935 		reg = rd32(hw, I40E_QRX_ENA(index));
3936 		reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
3937 		wr32(hw, I40E_QRX_ENA(index), reg);
3938 		/* Verify the disable took */
3939 		for (int j = 0; j < 10; j++) {
3940 			reg = rd32(hw, I40E_QRX_ENA(index));
3941 			if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
3942 				break;
3943 			i40e_msec_delay(10);
3944 		}
3945 		if (reg & I40E_QRX_ENA_QENA_STAT_MASK) {
3946 			device_printf(pf->dev, "RX queue %d still enabled!\n",
3947 			    index);
3948 			error = ETIMEDOUT;
3949 		}
3950 	}
3951 
3952 	return (error);
3953 }
3954 
3955 /**
3956  * ixl_handle_mdd_event
3957  *
3958  * Called from interrupt handler to identify possibly malicious vfs
3959  * (But also detects events from the PF, as well)
3960  **/
3961 static void ixl_handle_mdd_event(struct ixl_pf *pf)
3962 {
3963 	struct i40e_hw *hw = &pf->hw;
3964 	device_t dev = pf->dev;
3965 	bool mdd_detected = false;
3966 	bool pf_mdd_detected = false;
3967 	u32 reg;
3968 
3969 	/* find what triggered the MDD event */
3970 	reg = rd32(hw, I40E_GL_MDET_TX);
3971 	if (reg & I40E_GL_MDET_TX_VALID_MASK) {
3972 		u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
3973 				I40E_GL_MDET_TX_PF_NUM_SHIFT;
3974 		u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
3975 				I40E_GL_MDET_TX_EVENT_SHIFT;
3976 		u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
3977 				I40E_GL_MDET_TX_QUEUE_SHIFT;
3978 		device_printf(dev,
3979 			 "Malicious Driver Detection event 0x%02x"
3980 			 " on TX queue %d pf number 0x%02x\n",
3981 			 event, queue, pf_num);
3982 		wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
3983 		mdd_detected = true;
3984 	}
3985 	reg = rd32(hw, I40E_GL_MDET_RX);
3986 	if (reg & I40E_GL_MDET_RX_VALID_MASK) {
3987 		u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
3988 				I40E_GL_MDET_RX_FUNCTION_SHIFT;
3989 		u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
3990 				I40E_GL_MDET_RX_EVENT_SHIFT;
3991 		u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
3992 				I40E_GL_MDET_RX_QUEUE_SHIFT;
3993 		device_printf(dev,
3994 			 "Malicious Driver Detection event 0x%02x"
3995 			 " on RX queue %d of function 0x%02x\n",
3996 			 event, queue, func);
3997 		wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
3998 		mdd_detected = true;
3999 	}
4000 
4001 	if (mdd_detected) {
4002 		reg = rd32(hw, I40E_PF_MDET_TX);
4003 		if (reg & I40E_PF_MDET_TX_VALID_MASK) {
4004 			wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
4005 			device_printf(dev,
4006 				 "MDD TX event is for this function 0x%08x",
4007 				 reg);
4008 			pf_mdd_detected = true;
4009 		}
4010 		reg = rd32(hw, I40E_PF_MDET_RX);
4011 		if (reg & I40E_PF_MDET_RX_VALID_MASK) {
4012 			wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
4013 			device_printf(dev,
4014 				 "MDD RX event is for this function 0x%08x",
4015 				 reg);
4016 			pf_mdd_detected = true;
4017 		}
4018 	}
4019 
4020 	/* re-enable mdd interrupt cause */
4021 	reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4022 	reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
4023 	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4024 	ixl_flush(hw);
4025 }
4026 
4027 static void
4028 ixl_enable_intr(struct ixl_vsi *vsi)
4029 {
4030 	struct i40e_hw		*hw = vsi->hw;
4031 	struct ixl_queue	*que = vsi->queues;
4032 
4033 	if (ixl_enable_msix) {
4034 		ixl_enable_adminq(hw);
4035 		for (int i = 0; i < vsi->num_queues; i++, que++)
4036 			ixl_enable_queue(hw, que->me);
4037 	} else
4038 		ixl_enable_legacy(hw);
4039 }
4040 
4041 static void
4042 ixl_disable_rings_intr(struct ixl_vsi *vsi)
4043 {
4044 	struct i40e_hw		*hw = vsi->hw;
4045 	struct ixl_queue	*que = vsi->queues;
4046 
4047 	for (int i = 0; i < vsi->num_queues; i++, que++)
4048 		ixl_disable_queue(hw, que->me);
4049 }
4050 
4051 static void
4052 ixl_disable_intr(struct ixl_vsi *vsi)
4053 {
4054 	struct i40e_hw		*hw = vsi->hw;
4055 
4056 	if (ixl_enable_msix)
4057 		ixl_disable_adminq(hw);
4058 	else
4059 		ixl_disable_legacy(hw);
4060 }
4061 
4062 static void
4063 ixl_enable_adminq(struct i40e_hw *hw)
4064 {
4065 	u32		reg;
4066 
4067 	reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4068 	    I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4069 	    (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4070 	wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4071 	ixl_flush(hw);
4072 	return;
4073 }
4074 
4075 static void
4076 ixl_disable_adminq(struct i40e_hw *hw)
4077 {
4078 	u32		reg;
4079 
4080 	reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4081 	wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4082 
4083 	return;
4084 }
4085 
4086 static void
4087 ixl_enable_queue(struct i40e_hw *hw, int id)
4088 {
4089 	u32		reg;
4090 
4091 	reg = I40E_PFINT_DYN_CTLN_INTENA_MASK |
4092 	    I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
4093 	    (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
4094 	wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4095 }
4096 
4097 static void
4098 ixl_disable_queue(struct i40e_hw *hw, int id)
4099 {
4100 	u32		reg;
4101 
4102 	reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT;
4103 	wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4104 
4105 	return;
4106 }
4107 
4108 static void
4109 ixl_enable_legacy(struct i40e_hw *hw)
4110 {
4111 	u32		reg;
4112 	reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4113 	    I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4114 	    (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4115 	wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4116 }
4117 
4118 static void
4119 ixl_disable_legacy(struct i40e_hw *hw)
4120 {
4121 	u32		reg;
4122 
4123 	reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4124 	wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4125 
4126 	return;
4127 }
4128 
4129 static void
4130 ixl_update_stats_counters(struct ixl_pf *pf)
4131 {
4132 	struct i40e_hw	*hw = &pf->hw;
4133 	struct ixl_vsi	*vsi = &pf->vsi;
4134 	struct ixl_vf	*vf;
4135 
4136 	struct i40e_hw_port_stats *nsd = &pf->stats;
4137 	struct i40e_hw_port_stats *osd = &pf->stats_offsets;
4138 
4139 	/* Update hw stats */
4140 	ixl_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
4141 			   pf->stat_offsets_loaded,
4142 			   &osd->crc_errors, &nsd->crc_errors);
4143 	ixl_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
4144 			   pf->stat_offsets_loaded,
4145 			   &osd->illegal_bytes, &nsd->illegal_bytes);
4146 	ixl_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
4147 			   I40E_GLPRT_GORCL(hw->port),
4148 			   pf->stat_offsets_loaded,
4149 			   &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
4150 	ixl_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
4151 			   I40E_GLPRT_GOTCL(hw->port),
4152 			   pf->stat_offsets_loaded,
4153 			   &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
4154 	ixl_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
4155 			   pf->stat_offsets_loaded,
4156 			   &osd->eth.rx_discards,
4157 			   &nsd->eth.rx_discards);
4158 	ixl_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
4159 			   I40E_GLPRT_UPRCL(hw->port),
4160 			   pf->stat_offsets_loaded,
4161 			   &osd->eth.rx_unicast,
4162 			   &nsd->eth.rx_unicast);
4163 	ixl_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
4164 			   I40E_GLPRT_UPTCL(hw->port),
4165 			   pf->stat_offsets_loaded,
4166 			   &osd->eth.tx_unicast,
4167 			   &nsd->eth.tx_unicast);
4168 	ixl_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
4169 			   I40E_GLPRT_MPRCL(hw->port),
4170 			   pf->stat_offsets_loaded,
4171 			   &osd->eth.rx_multicast,
4172 			   &nsd->eth.rx_multicast);
4173 	ixl_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
4174 			   I40E_GLPRT_MPTCL(hw->port),
4175 			   pf->stat_offsets_loaded,
4176 			   &osd->eth.tx_multicast,
4177 			   &nsd->eth.tx_multicast);
4178 	ixl_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
4179 			   I40E_GLPRT_BPRCL(hw->port),
4180 			   pf->stat_offsets_loaded,
4181 			   &osd->eth.rx_broadcast,
4182 			   &nsd->eth.rx_broadcast);
4183 	ixl_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
4184 			   I40E_GLPRT_BPTCL(hw->port),
4185 			   pf->stat_offsets_loaded,
4186 			   &osd->eth.tx_broadcast,
4187 			   &nsd->eth.tx_broadcast);
4188 
4189 	ixl_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
4190 			   pf->stat_offsets_loaded,
4191 			   &osd->tx_dropped_link_down,
4192 			   &nsd->tx_dropped_link_down);
4193 	ixl_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
4194 			   pf->stat_offsets_loaded,
4195 			   &osd->mac_local_faults,
4196 			   &nsd->mac_local_faults);
4197 	ixl_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
4198 			   pf->stat_offsets_loaded,
4199 			   &osd->mac_remote_faults,
4200 			   &nsd->mac_remote_faults);
4201 	ixl_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
4202 			   pf->stat_offsets_loaded,
4203 			   &osd->rx_length_errors,
4204 			   &nsd->rx_length_errors);
4205 
4206 	/* Flow control (LFC) stats */
4207 	ixl_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
4208 			   pf->stat_offsets_loaded,
4209 			   &osd->link_xon_rx, &nsd->link_xon_rx);
4210 	ixl_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
4211 			   pf->stat_offsets_loaded,
4212 			   &osd->link_xon_tx, &nsd->link_xon_tx);
4213 	ixl_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
4214 			   pf->stat_offsets_loaded,
4215 			   &osd->link_xoff_rx, &nsd->link_xoff_rx);
4216 	ixl_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
4217 			   pf->stat_offsets_loaded,
4218 			   &osd->link_xoff_tx, &nsd->link_xoff_tx);
4219 
4220 	/* Packet size stats rx */
4221 	ixl_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
4222 			   I40E_GLPRT_PRC64L(hw->port),
4223 			   pf->stat_offsets_loaded,
4224 			   &osd->rx_size_64, &nsd->rx_size_64);
4225 	ixl_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
4226 			   I40E_GLPRT_PRC127L(hw->port),
4227 			   pf->stat_offsets_loaded,
4228 			   &osd->rx_size_127, &nsd->rx_size_127);
4229 	ixl_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
4230 			   I40E_GLPRT_PRC255L(hw->port),
4231 			   pf->stat_offsets_loaded,
4232 			   &osd->rx_size_255, &nsd->rx_size_255);
4233 	ixl_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
4234 			   I40E_GLPRT_PRC511L(hw->port),
4235 			   pf->stat_offsets_loaded,
4236 			   &osd->rx_size_511, &nsd->rx_size_511);
4237 	ixl_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
4238 			   I40E_GLPRT_PRC1023L(hw->port),
4239 			   pf->stat_offsets_loaded,
4240 			   &osd->rx_size_1023, &nsd->rx_size_1023);
4241 	ixl_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
4242 			   I40E_GLPRT_PRC1522L(hw->port),
4243 			   pf->stat_offsets_loaded,
4244 			   &osd->rx_size_1522, &nsd->rx_size_1522);
4245 	ixl_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
4246 			   I40E_GLPRT_PRC9522L(hw->port),
4247 			   pf->stat_offsets_loaded,
4248 			   &osd->rx_size_big, &nsd->rx_size_big);
4249 
4250 	/* Packet size stats tx */
4251 	ixl_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
4252 			   I40E_GLPRT_PTC64L(hw->port),
4253 			   pf->stat_offsets_loaded,
4254 			   &osd->tx_size_64, &nsd->tx_size_64);
4255 	ixl_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
4256 			   I40E_GLPRT_PTC127L(hw->port),
4257 			   pf->stat_offsets_loaded,
4258 			   &osd->tx_size_127, &nsd->tx_size_127);
4259 	ixl_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
4260 			   I40E_GLPRT_PTC255L(hw->port),
4261 			   pf->stat_offsets_loaded,
4262 			   &osd->tx_size_255, &nsd->tx_size_255);
4263 	ixl_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
4264 			   I40E_GLPRT_PTC511L(hw->port),
4265 			   pf->stat_offsets_loaded,
4266 			   &osd->tx_size_511, &nsd->tx_size_511);
4267 	ixl_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
4268 			   I40E_GLPRT_PTC1023L(hw->port),
4269 			   pf->stat_offsets_loaded,
4270 			   &osd->tx_size_1023, &nsd->tx_size_1023);
4271 	ixl_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
4272 			   I40E_GLPRT_PTC1522L(hw->port),
4273 			   pf->stat_offsets_loaded,
4274 			   &osd->tx_size_1522, &nsd->tx_size_1522);
4275 	ixl_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
4276 			   I40E_GLPRT_PTC9522L(hw->port),
4277 			   pf->stat_offsets_loaded,
4278 			   &osd->tx_size_big, &nsd->tx_size_big);
4279 
4280 	ixl_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
4281 			   pf->stat_offsets_loaded,
4282 			   &osd->rx_undersize, &nsd->rx_undersize);
4283 	ixl_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
4284 			   pf->stat_offsets_loaded,
4285 			   &osd->rx_fragments, &nsd->rx_fragments);
4286 	ixl_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
4287 			   pf->stat_offsets_loaded,
4288 			   &osd->rx_oversize, &nsd->rx_oversize);
4289 	ixl_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
4290 			   pf->stat_offsets_loaded,
4291 			   &osd->rx_jabber, &nsd->rx_jabber);
4292 	pf->stat_offsets_loaded = true;
4293 	/* End hw stats */
4294 
4295 	/* Update vsi stats */
4296 	ixl_update_vsi_stats(vsi);
4297 
4298 	for (int i = 0; i < pf->num_vfs; i++) {
4299 		vf = &pf->vfs[i];
4300 		if (vf->vf_flags & VF_FLAG_ENABLED)
4301 			ixl_update_eth_stats(&pf->vfs[i].vsi);
4302 	}
4303 }
4304 
4305 /*
4306 ** Tasklet handler for MSIX Adminq interrupts
4307 **  - do outside interrupt since it might sleep
4308 */
4309 static void
4310 ixl_do_adminq(void *context, int pending)
4311 {
4312 	struct ixl_pf			*pf = context;
4313 	struct i40e_hw			*hw = &pf->hw;
4314 	struct ixl_vsi			*vsi = &pf->vsi;
4315 	struct i40e_arq_event_info	event;
4316 	i40e_status			ret;
4317 	u32				reg, loop = 0;
4318 	u16				opcode, result;
4319 
4320 	event.buf_len = IXL_AQ_BUF_SZ;
4321 	event.msg_buf = malloc(event.buf_len,
4322 	    M_DEVBUF, M_NOWAIT | M_ZERO);
4323 	if (!event.msg_buf) {
4324 		printf("Unable to allocate adminq memory\n");
4325 		return;
4326 	}
4327 
4328 	IXL_PF_LOCK(pf);
4329 	/* clean and process any events */
4330 	do {
4331 		ret = i40e_clean_arq_element(hw, &event, &result);
4332 		if (ret)
4333 			break;
4334 		opcode = LE16_TO_CPU(event.desc.opcode);
4335 		switch (opcode) {
4336 		case i40e_aqc_opc_get_link_status:
4337 			ixl_link_event(pf, &event);
4338 			ixl_update_link_status(pf);
4339 			break;
4340 		case i40e_aqc_opc_send_msg_to_pf:
4341 #ifdef PCI_IOV
4342 			ixl_handle_vf_msg(pf, &event);
4343 #endif
4344 			break;
4345 		case i40e_aqc_opc_event_lan_overflow:
4346 			break;
4347 		default:
4348 #ifdef IXL_DEBUG
4349 			printf("AdminQ unknown event %x\n", opcode);
4350 #endif
4351 			break;
4352 		}
4353 
4354 	} while (result && (loop++ < IXL_ADM_LIMIT));
4355 
4356 	reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4357 	reg |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
4358 	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4359 	free(event.msg_buf, M_DEVBUF);
4360 
4361 	/*
4362 	 * If there are still messages to process, reschedule ourselves.
4363 	 * Otherwise, re-enable our interrupt and go to sleep.
4364 	 */
4365 	if (result > 0)
4366 		taskqueue_enqueue(pf->tq, &pf->adminq);
4367 	else
4368 		ixl_enable_intr(vsi);
4369 
4370 	IXL_PF_UNLOCK(pf);
4371 }
4372 
4373 static int
4374 ixl_debug_info(SYSCTL_HANDLER_ARGS)
4375 {
4376 	struct ixl_pf	*pf;
4377 	int		error, input = 0;
4378 
4379 	error = sysctl_handle_int(oidp, &input, 0, req);
4380 
4381 	if (error || !req->newptr)
4382 		return (error);
4383 
4384 	if (input == 1) {
4385 		pf = (struct ixl_pf *)arg1;
4386 		ixl_print_debug_info(pf);
4387 	}
4388 
4389 	return (error);
4390 }
4391 
4392 static void
4393 ixl_print_debug_info(struct ixl_pf *pf)
4394 {
4395 	struct i40e_hw		*hw = &pf->hw;
4396 	struct ixl_vsi		*vsi = &pf->vsi;
4397 	struct ixl_queue	*que = vsi->queues;
4398 	struct rx_ring		*rxr = &que->rxr;
4399 	struct tx_ring		*txr = &que->txr;
4400 	u32			reg;
4401 
4402 
4403 	printf("Queue irqs = %jx\n", (uintmax_t)que->irqs);
4404 	printf("AdminQ irqs = %jx\n", (uintmax_t)pf->admin_irq);
4405 	printf("RX next check = %x\n", rxr->next_check);
4406 	printf("RX not ready = %jx\n", (uintmax_t)rxr->not_done);
4407 	printf("RX packets = %jx\n", (uintmax_t)rxr->rx_packets);
4408 	printf("TX desc avail = %x\n", txr->avail);
4409 
4410 	reg = rd32(hw, I40E_GLV_GORCL(0xc));
4411 	 printf("RX Bytes = %x\n", reg);
4412 	reg = rd32(hw, I40E_GLPRT_GORCL(hw->port));
4413 	 printf("Port RX Bytes = %x\n", reg);
4414 	reg = rd32(hw, I40E_GLV_RDPC(0xc));
4415 	 printf("RX discard = %x\n", reg);
4416 	reg = rd32(hw, I40E_GLPRT_RDPC(hw->port));
4417 	 printf("Port RX discard = %x\n", reg);
4418 
4419 	reg = rd32(hw, I40E_GLV_TEPC(0xc));
4420 	 printf("TX errors = %x\n", reg);
4421 	reg = rd32(hw, I40E_GLV_GOTCL(0xc));
4422 	 printf("TX Bytes = %x\n", reg);
4423 
4424 	reg = rd32(hw, I40E_GLPRT_RUC(hw->port));
4425 	 printf("RX undersize = %x\n", reg);
4426 	reg = rd32(hw, I40E_GLPRT_RFC(hw->port));
4427 	 printf("RX fragments = %x\n", reg);
4428 	reg = rd32(hw, I40E_GLPRT_ROC(hw->port));
4429 	 printf("RX oversize = %x\n", reg);
4430 	reg = rd32(hw, I40E_GLPRT_RLEC(hw->port));
4431 	 printf("RX length error = %x\n", reg);
4432 	reg = rd32(hw, I40E_GLPRT_MRFC(hw->port));
4433 	 printf("mac remote fault = %x\n", reg);
4434 	reg = rd32(hw, I40E_GLPRT_MLFC(hw->port));
4435 	 printf("mac local fault = %x\n", reg);
4436 }
4437 
4438 /**
4439  * Update VSI-specific ethernet statistics counters.
4440  **/
4441 void ixl_update_eth_stats(struct ixl_vsi *vsi)
4442 {
4443 	struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
4444 	struct i40e_hw *hw = &pf->hw;
4445 	struct i40e_eth_stats *es;
4446 	struct i40e_eth_stats *oes;
4447 	struct i40e_hw_port_stats *nsd;
4448 	u16 stat_idx = vsi->info.stat_counter_idx;
4449 
4450 	es = &vsi->eth_stats;
4451 	oes = &vsi->eth_stats_offsets;
4452 	nsd = &pf->stats;
4453 
4454 	/* Gather up the stats that the hw collects */
4455 	ixl_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
4456 			   vsi->stat_offsets_loaded,
4457 			   &oes->tx_errors, &es->tx_errors);
4458 	ixl_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
4459 			   vsi->stat_offsets_loaded,
4460 			   &oes->rx_discards, &es->rx_discards);
4461 
4462 	ixl_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
4463 			   I40E_GLV_GORCL(stat_idx),
4464 			   vsi->stat_offsets_loaded,
4465 			   &oes->rx_bytes, &es->rx_bytes);
4466 	ixl_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
4467 			   I40E_GLV_UPRCL(stat_idx),
4468 			   vsi->stat_offsets_loaded,
4469 			   &oes->rx_unicast, &es->rx_unicast);
4470 	ixl_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
4471 			   I40E_GLV_MPRCL(stat_idx),
4472 			   vsi->stat_offsets_loaded,
4473 			   &oes->rx_multicast, &es->rx_multicast);
4474 	ixl_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
4475 			   I40E_GLV_BPRCL(stat_idx),
4476 			   vsi->stat_offsets_loaded,
4477 			   &oes->rx_broadcast, &es->rx_broadcast);
4478 
4479 	ixl_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
4480 			   I40E_GLV_GOTCL(stat_idx),
4481 			   vsi->stat_offsets_loaded,
4482 			   &oes->tx_bytes, &es->tx_bytes);
4483 	ixl_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
4484 			   I40E_GLV_UPTCL(stat_idx),
4485 			   vsi->stat_offsets_loaded,
4486 			   &oes->tx_unicast, &es->tx_unicast);
4487 	ixl_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
4488 			   I40E_GLV_MPTCL(stat_idx),
4489 			   vsi->stat_offsets_loaded,
4490 			   &oes->tx_multicast, &es->tx_multicast);
4491 	ixl_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
4492 			   I40E_GLV_BPTCL(stat_idx),
4493 			   vsi->stat_offsets_loaded,
4494 			   &oes->tx_broadcast, &es->tx_broadcast);
4495 	vsi->stat_offsets_loaded = true;
4496 }
4497 
4498 static void
4499 ixl_update_vsi_stats(struct ixl_vsi *vsi)
4500 {
4501 	struct ixl_pf		*pf;
4502 	struct ifnet		*ifp;
4503 	struct i40e_eth_stats	*es;
4504 	u64			tx_discards;
4505 
4506 	struct i40e_hw_port_stats *nsd;
4507 
4508 	pf = vsi->back;
4509 	ifp = vsi->ifp;
4510 	es = &vsi->eth_stats;
4511 	nsd = &pf->stats;
4512 
4513 	ixl_update_eth_stats(vsi);
4514 
4515 	tx_discards = es->tx_discards + nsd->tx_dropped_link_down;
4516 	for (int i = 0; i < vsi->num_queues; i++)
4517 		tx_discards += vsi->queues[i].txr.br->br_drops;
4518 
4519 	/* Update ifnet stats */
4520 	IXL_SET_IPACKETS(vsi, es->rx_unicast +
4521 	                   es->rx_multicast +
4522 			   es->rx_broadcast);
4523 	IXL_SET_OPACKETS(vsi, es->tx_unicast +
4524 	                   es->tx_multicast +
4525 			   es->tx_broadcast);
4526 	IXL_SET_IBYTES(vsi, es->rx_bytes);
4527 	IXL_SET_OBYTES(vsi, es->tx_bytes);
4528 	IXL_SET_IMCASTS(vsi, es->rx_multicast);
4529 	IXL_SET_OMCASTS(vsi, es->tx_multicast);
4530 
4531 	IXL_SET_IERRORS(vsi, nsd->crc_errors + nsd->illegal_bytes +
4532 	    nsd->rx_undersize + nsd->rx_oversize + nsd->rx_fragments +
4533 	    nsd->rx_jabber);
4534 	IXL_SET_OERRORS(vsi, es->tx_errors);
4535 	IXL_SET_IQDROPS(vsi, es->rx_discards + nsd->eth.rx_discards);
4536 	IXL_SET_OQDROPS(vsi, tx_discards);
4537 	IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol);
4538 	IXL_SET_COLLISIONS(vsi, 0);
4539 }
4540 
4541 /**
4542  * Reset all of the stats for the given pf
4543  **/
4544 void ixl_pf_reset_stats(struct ixl_pf *pf)
4545 {
4546 	bzero(&pf->stats, sizeof(struct i40e_hw_port_stats));
4547 	bzero(&pf->stats_offsets, sizeof(struct i40e_hw_port_stats));
4548 	pf->stat_offsets_loaded = false;
4549 }
4550 
4551 /**
4552  * Resets all stats of the given vsi
4553  **/
4554 void ixl_vsi_reset_stats(struct ixl_vsi *vsi)
4555 {
4556 	bzero(&vsi->eth_stats, sizeof(struct i40e_eth_stats));
4557 	bzero(&vsi->eth_stats_offsets, sizeof(struct i40e_eth_stats));
4558 	vsi->stat_offsets_loaded = false;
4559 }
4560 
4561 /**
4562  * Read and update a 48 bit stat from the hw
4563  *
4564  * Since the device stats are not reset at PFReset, they likely will not
4565  * be zeroed when the driver starts.  We'll save the first values read
4566  * and use them as offsets to be subtracted from the raw values in order
4567  * to report stats that count from zero.
4568  **/
4569 static void
4570 ixl_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
4571 	bool offset_loaded, u64 *offset, u64 *stat)
4572 {
4573 	u64 new_data;
4574 
4575 #if defined(__FreeBSD__) && (__FreeBSD_version >= 1000000) && defined(__amd64__)
4576 	new_data = rd64(hw, loreg);
4577 #else
4578 	/*
4579 	 * Use two rd32's instead of one rd64; FreeBSD versions before
4580 	 * 10 don't support 8 byte bus reads/writes.
4581 	 */
4582 	new_data = rd32(hw, loreg);
4583 	new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
4584 #endif
4585 
4586 	if (!offset_loaded)
4587 		*offset = new_data;
4588 	if (new_data >= *offset)
4589 		*stat = new_data - *offset;
4590 	else
4591 		*stat = (new_data + ((u64)1 << 48)) - *offset;
4592 	*stat &= 0xFFFFFFFFFFFFULL;
4593 }
4594 
4595 /**
4596  * Read and update a 32 bit stat from the hw
4597  **/
4598 static void
4599 ixl_stat_update32(struct i40e_hw *hw, u32 reg,
4600 	bool offset_loaded, u64 *offset, u64 *stat)
4601 {
4602 	u32 new_data;
4603 
4604 	new_data = rd32(hw, reg);
4605 	if (!offset_loaded)
4606 		*offset = new_data;
4607 	if (new_data >= *offset)
4608 		*stat = (u32)(new_data - *offset);
4609 	else
4610 		*stat = (u32)((new_data + ((u64)1 << 32)) - *offset);
4611 }
4612 
4613 /*
4614 ** Set flow control using sysctl:
4615 ** 	0 - off
4616 **	1 - rx pause
4617 **	2 - tx pause
4618 **	3 - full
4619 */
4620 static int
4621 ixl_set_flowcntl(SYSCTL_HANDLER_ARGS)
4622 {
4623 	/*
4624 	 * TODO: ensure flow control is disabled if
4625 	 * priority flow control is enabled
4626 	 *
4627 	 * TODO: ensure tx CRC by hardware should be enabled
4628 	 * if tx flow control is enabled.
4629 	 */
4630 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
4631 	struct i40e_hw *hw = &pf->hw;
4632 	device_t dev = pf->dev;
4633 	int error = 0;
4634 	enum i40e_status_code aq_error = 0;
4635 	u8 fc_aq_err = 0;
4636 
4637 	/* Get request */
4638 	error = sysctl_handle_int(oidp, &pf->fc, 0, req);
4639 	if ((error) || (req->newptr == NULL))
4640 		return (error);
4641 	if (pf->fc < 0 || pf->fc > 3) {
4642 		device_printf(dev,
4643 		    "Invalid fc mode; valid modes are 0 through 3\n");
4644 		return (EINVAL);
4645 	}
4646 
4647 	/*
4648 	** Changing flow control mode currently does not work on
4649 	** 40GBASE-CR4 PHYs
4650 	*/
4651 	if (hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4
4652 	    || hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4_CU) {
4653 		device_printf(dev, "Changing flow control mode unsupported"
4654 		    " on 40GBase-CR4 media.\n");
4655 		return (ENODEV);
4656 	}
4657 
4658 	/* Set fc ability for port */
4659 	hw->fc.requested_mode = pf->fc;
4660 	aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE);
4661 	if (aq_error) {
4662 		device_printf(dev,
4663 		    "%s: Error setting new fc mode %d; fc_err %#x\n",
4664 		    __func__, aq_error, fc_aq_err);
4665 		return (EAGAIN);
4666 	}
4667 
4668 	return (0);
4669 }
4670 
4671 static int
4672 ixl_current_speed(SYSCTL_HANDLER_ARGS)
4673 {
4674 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
4675 	struct i40e_hw *hw = &pf->hw;
4676 	int error = 0, index = 0;
4677 
4678 	char *speeds[] = {
4679 		"Unknown",
4680 		"100M",
4681 		"1G",
4682 		"10G",
4683 		"40G",
4684 		"20G"
4685 	};
4686 
4687 	ixl_update_link_status(pf);
4688 
4689 	switch (hw->phy.link_info.link_speed) {
4690 	case I40E_LINK_SPEED_100MB:
4691 		index = 1;
4692 		break;
4693 	case I40E_LINK_SPEED_1GB:
4694 		index = 2;
4695 		break;
4696 	case I40E_LINK_SPEED_10GB:
4697 		index = 3;
4698 		break;
4699 	case I40E_LINK_SPEED_40GB:
4700 		index = 4;
4701 		break;
4702 	case I40E_LINK_SPEED_20GB:
4703 		index = 5;
4704 		break;
4705 	case I40E_LINK_SPEED_UNKNOWN:
4706 	default:
4707 		index = 0;
4708 		break;
4709 	}
4710 
4711 	error = sysctl_handle_string(oidp, speeds[index],
4712 	    strlen(speeds[index]), req);
4713 	return (error);
4714 }
4715 
4716 static int
4717 ixl_set_advertised_speeds(struct ixl_pf *pf, int speeds)
4718 {
4719 	struct i40e_hw *hw = &pf->hw;
4720 	device_t dev = pf->dev;
4721 	struct i40e_aq_get_phy_abilities_resp abilities;
4722 	struct i40e_aq_set_phy_config config;
4723 	enum i40e_status_code aq_error = 0;
4724 
4725 	/* Get current capability information */
4726 	aq_error = i40e_aq_get_phy_capabilities(hw,
4727 	    FALSE, FALSE, &abilities, NULL);
4728 	if (aq_error) {
4729 		device_printf(dev,
4730 		    "%s: Error getting phy capabilities %d,"
4731 		    " aq error: %d\n", __func__, aq_error,
4732 		    hw->aq.asq_last_status);
4733 		return (EAGAIN);
4734 	}
4735 
4736 	/* Prepare new config */
4737 	bzero(&config, sizeof(config));
4738 	config.phy_type = abilities.phy_type;
4739 	config.abilities = abilities.abilities
4740 	    | I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
4741 	config.eee_capability = abilities.eee_capability;
4742 	config.eeer = abilities.eeer_val;
4743 	config.low_power_ctrl = abilities.d3_lpan;
4744 	/* Translate into aq cmd link_speed */
4745 	if (speeds & 0x8)
4746 		config.link_speed |= I40E_LINK_SPEED_20GB;
4747 	if (speeds & 0x4)
4748 		config.link_speed |= I40E_LINK_SPEED_10GB;
4749 	if (speeds & 0x2)
4750 		config.link_speed |= I40E_LINK_SPEED_1GB;
4751 	if (speeds & 0x1)
4752 		config.link_speed |= I40E_LINK_SPEED_100MB;
4753 
4754 	/* Do aq command & restart link */
4755 	aq_error = i40e_aq_set_phy_config(hw, &config, NULL);
4756 	if (aq_error) {
4757 		device_printf(dev,
4758 		    "%s: Error setting new phy config %d,"
4759 		    " aq error: %d\n", __func__, aq_error,
4760 		    hw->aq.asq_last_status);
4761 		return (EAGAIN);
4762 	}
4763 
4764 	/*
4765 	** This seems a bit heavy handed, but we
4766 	** need to get a reinit on some devices
4767 	*/
4768 	IXL_PF_LOCK(pf);
4769 	ixl_stop(pf);
4770 	ixl_init_locked(pf);
4771 	IXL_PF_UNLOCK(pf);
4772 
4773 	return (0);
4774 }
4775 
4776 /*
4777 ** Control link advertise speed:
4778 **	Flags:
4779 **	0x1 - advertise 100 Mb
4780 **	0x2 - advertise 1G
4781 **	0x4 - advertise 10G
4782 **	0x8 - advertise 20G
4783 **
4784 ** Does not work on 40G devices.
4785 */
4786 static int
4787 ixl_set_advertise(SYSCTL_HANDLER_ARGS)
4788 {
4789 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
4790 	struct i40e_hw *hw = &pf->hw;
4791 	device_t dev = pf->dev;
4792 	int requested_ls = 0;
4793 	int error = 0;
4794 
4795 	/*
4796 	** FW doesn't support changing advertised speed
4797 	** for 40G devices; speed is always 40G.
4798 	*/
4799 	if (i40e_is_40G_device(hw->device_id))
4800 		return (ENODEV);
4801 
4802 	/* Read in new mode */
4803 	requested_ls = pf->advertised_speed;
4804 	error = sysctl_handle_int(oidp, &requested_ls, 0, req);
4805 	if ((error) || (req->newptr == NULL))
4806 		return (error);
4807 	/* Check for sane value */
4808 	if (requested_ls < 0x1 || requested_ls > 0xE) {
4809 		device_printf(dev, "Invalid advertised speed; "
4810 		    "valid modes are 0x1 through 0xE\n");
4811 		return (EINVAL);
4812 	}
4813 	/* Then check for validity based on adapter type */
4814 	switch (hw->device_id) {
4815 	case I40E_DEV_ID_10G_BASE_T:
4816 		if (requested_ls & 0x8) {
4817 			device_printf(dev,
4818 			    "20Gbs speed not supported on this device.\n");
4819 			return (EINVAL);
4820 		}
4821 		break;
4822 	case I40E_DEV_ID_20G_KR2:
4823 		if (requested_ls & 0x1) {
4824 			device_printf(dev,
4825 			    "100Mbs speed not supported on this device.\n");
4826 			return (EINVAL);
4827 		}
4828 		break;
4829 	default:
4830 		if (requested_ls & ~0x6) {
4831 			device_printf(dev,
4832 			    "Only 1/10Gbs speeds are supported on this device.\n");
4833 			return (EINVAL);
4834 		}
4835 		break;
4836 	}
4837 
4838 	/* Exit if no change */
4839 	if (pf->advertised_speed == requested_ls)
4840 		return (0);
4841 
4842 	error = ixl_set_advertised_speeds(pf, requested_ls);
4843 	if (error)
4844 		return (error);
4845 
4846 	pf->advertised_speed = requested_ls;
4847 	ixl_update_link_status(pf);
4848 	return (0);
4849 }
4850 
4851 /*
4852 ** Get the width and transaction speed of
4853 ** the bus this adapter is plugged into.
4854 */
4855 static u16
4856 ixl_get_bus_info(struct i40e_hw *hw, device_t dev)
4857 {
4858         u16                     link;
4859         u32                     offset;
4860 
4861 
4862         /* Get the PCI Express Capabilities offset */
4863         pci_find_cap(dev, PCIY_EXPRESS, &offset);
4864 
4865         /* ...and read the Link Status Register */
4866         link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
4867 
4868         switch (link & I40E_PCI_LINK_WIDTH) {
4869         case I40E_PCI_LINK_WIDTH_1:
4870                 hw->bus.width = i40e_bus_width_pcie_x1;
4871                 break;
4872         case I40E_PCI_LINK_WIDTH_2:
4873                 hw->bus.width = i40e_bus_width_pcie_x2;
4874                 break;
4875         case I40E_PCI_LINK_WIDTH_4:
4876                 hw->bus.width = i40e_bus_width_pcie_x4;
4877                 break;
4878         case I40E_PCI_LINK_WIDTH_8:
4879                 hw->bus.width = i40e_bus_width_pcie_x8;
4880                 break;
4881         default:
4882                 hw->bus.width = i40e_bus_width_unknown;
4883                 break;
4884         }
4885 
4886         switch (link & I40E_PCI_LINK_SPEED) {
4887         case I40E_PCI_LINK_SPEED_2500:
4888                 hw->bus.speed = i40e_bus_speed_2500;
4889                 break;
4890         case I40E_PCI_LINK_SPEED_5000:
4891                 hw->bus.speed = i40e_bus_speed_5000;
4892                 break;
4893         case I40E_PCI_LINK_SPEED_8000:
4894                 hw->bus.speed = i40e_bus_speed_8000;
4895                 break;
4896         default:
4897                 hw->bus.speed = i40e_bus_speed_unknown;
4898                 break;
4899         }
4900 
4901 
4902         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
4903             ((hw->bus.speed == i40e_bus_speed_8000) ? "8.0GT/s":
4904             (hw->bus.speed == i40e_bus_speed_5000) ? "5.0GT/s":
4905             (hw->bus.speed == i40e_bus_speed_2500) ? "2.5GT/s":"Unknown"),
4906             (hw->bus.width == i40e_bus_width_pcie_x8) ? "Width x8" :
4907             (hw->bus.width == i40e_bus_width_pcie_x4) ? "Width x4" :
4908             (hw->bus.width == i40e_bus_width_pcie_x1) ? "Width x1" :
4909             ("Unknown"));
4910 
4911         if ((hw->bus.width <= i40e_bus_width_pcie_x8) &&
4912             (hw->bus.speed < i40e_bus_speed_8000)) {
4913                 device_printf(dev, "PCI-Express bandwidth available"
4914                     " for this device\n     may be insufficient for"
4915                     " optimal performance.\n");
4916                 device_printf(dev, "For expected performance a x8 "
4917                     "PCIE Gen3 slot is required.\n");
4918         }
4919 
4920         return (link);
4921 }
4922 
4923 static int
4924 ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS)
4925 {
4926 	struct ixl_pf	*pf = (struct ixl_pf *)arg1;
4927 	struct i40e_hw	*hw = &pf->hw;
4928 	char		buf[32];
4929 
4930 	snprintf(buf, sizeof(buf),
4931 	    "f%d.%d a%d.%d n%02x.%02x e%08x",
4932 	    hw->aq.fw_maj_ver, hw->aq.fw_min_ver,
4933 	    hw->aq.api_maj_ver, hw->aq.api_min_ver,
4934 	    (hw->nvm.version & IXL_NVM_VERSION_HI_MASK) >>
4935 	    IXL_NVM_VERSION_HI_SHIFT,
4936 	    (hw->nvm.version & IXL_NVM_VERSION_LO_MASK) >>
4937 	    IXL_NVM_VERSION_LO_SHIFT,
4938 	    hw->nvm.eetrack);
4939 	return (sysctl_handle_string(oidp, buf, strlen(buf), req));
4940 }
4941 
4942 
4943 #ifdef IXL_DEBUG_SYSCTL
4944 static int
4945 ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS)
4946 {
4947 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
4948 	struct i40e_hw *hw = &pf->hw;
4949 	struct i40e_link_status link_status;
4950 	char buf[512];
4951 
4952 	enum i40e_status_code aq_error = 0;
4953 
4954 	aq_error = i40e_aq_get_link_info(hw, TRUE, &link_status, NULL);
4955 	if (aq_error) {
4956 		printf("i40e_aq_get_link_info() error %d\n", aq_error);
4957 		return (EPERM);
4958 	}
4959 
4960 	sprintf(buf, "\n"
4961 	    "PHY Type : %#04x\n"
4962 	    "Speed    : %#04x\n"
4963 	    "Link info: %#04x\n"
4964 	    "AN info  : %#04x\n"
4965 	    "Ext info : %#04x",
4966 	    link_status.phy_type, link_status.link_speed,
4967 	    link_status.link_info, link_status.an_info,
4968 	    link_status.ext_info);
4969 
4970 	return (sysctl_handle_string(oidp, buf, strlen(buf), req));
4971 }
4972 
4973 static int
4974 ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS)
4975 {
4976 	struct ixl_pf		*pf = (struct ixl_pf *)arg1;
4977 	struct i40e_hw		*hw = &pf->hw;
4978 	char			buf[512];
4979 	enum i40e_status_code	aq_error = 0;
4980 
4981 	struct i40e_aq_get_phy_abilities_resp abilities;
4982 
4983 	aq_error = i40e_aq_get_phy_capabilities(hw,
4984 	    TRUE, FALSE, &abilities, NULL);
4985 	if (aq_error) {
4986 		printf("i40e_aq_get_phy_capabilities() error %d\n", aq_error);
4987 		return (EPERM);
4988 	}
4989 
4990 	sprintf(buf, "\n"
4991 	    "PHY Type : %#010x\n"
4992 	    "Speed    : %#04x\n"
4993 	    "Abilities: %#04x\n"
4994 	    "EEE cap  : %#06x\n"
4995 	    "EEER reg : %#010x\n"
4996 	    "D3 Lpan  : %#04x",
4997 	    abilities.phy_type, abilities.link_speed,
4998 	    abilities.abilities, abilities.eee_capability,
4999 	    abilities.eeer_val, abilities.d3_lpan);
5000 
5001 	return (sysctl_handle_string(oidp, buf, strlen(buf), req));
5002 }
5003 
5004 static int
5005 ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS)
5006 {
5007 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
5008 	struct ixl_vsi *vsi = &pf->vsi;
5009 	struct ixl_mac_filter *f;
5010 	char *buf, *buf_i;
5011 
5012 	int error = 0;
5013 	int ftl_len = 0;
5014 	int ftl_counter = 0;
5015 	int buf_len = 0;
5016 	int entry_len = 42;
5017 
5018 	SLIST_FOREACH(f, &vsi->ftl, next) {
5019 		ftl_len++;
5020 	}
5021 
5022 	if (ftl_len < 1) {
5023 		sysctl_handle_string(oidp, "(none)", 6, req);
5024 		return (0);
5025 	}
5026 
5027 	buf_len = sizeof(char) * (entry_len + 1) * ftl_len + 2;
5028 	buf = buf_i = malloc(buf_len, M_DEVBUF, M_NOWAIT);
5029 
5030 	sprintf(buf_i++, "\n");
5031 	SLIST_FOREACH(f, &vsi->ftl, next) {
5032 		sprintf(buf_i,
5033 		    MAC_FORMAT ", vlan %4d, flags %#06x",
5034 		    MAC_FORMAT_ARGS(f->macaddr), f->vlan, f->flags);
5035 		buf_i += entry_len;
5036 		/* don't print '\n' for last entry */
5037 		if (++ftl_counter != ftl_len) {
5038 			sprintf(buf_i, "\n");
5039 			buf_i++;
5040 		}
5041 	}
5042 
5043 	error = sysctl_handle_string(oidp, buf, strlen(buf), req);
5044 	if (error)
5045 		printf("sysctl error: %d\n", error);
5046 	free(buf, M_DEVBUF);
5047 	return error;
5048 }
5049 
5050 #define IXL_SW_RES_SIZE 0x14
5051 static int
5052 ixl_res_alloc_cmp(const void *a, const void *b)
5053 {
5054 	const struct i40e_aqc_switch_resource_alloc_element_resp *one, *two;
5055 	one = (const struct i40e_aqc_switch_resource_alloc_element_resp *)a;
5056 	two = (const struct i40e_aqc_switch_resource_alloc_element_resp *)b;
5057 
5058 	return ((int)one->resource_type - (int)two->resource_type);
5059 }
5060 
5061 static int
5062 ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS)
5063 {
5064 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
5065 	struct i40e_hw *hw = &pf->hw;
5066 	device_t dev = pf->dev;
5067 	struct sbuf *buf;
5068 	int error = 0;
5069 
5070 	u8 num_entries;
5071 	struct i40e_aqc_switch_resource_alloc_element_resp resp[IXL_SW_RES_SIZE];
5072 
5073 	buf = sbuf_new_for_sysctl(NULL, NULL, 0, req);
5074 	if (!buf) {
5075 		device_printf(dev, "Could not allocate sbuf for output.\n");
5076 		return (ENOMEM);
5077 	}
5078 
5079 	bzero(resp, sizeof(resp));
5080 	error = i40e_aq_get_switch_resource_alloc(hw, &num_entries,
5081 				resp,
5082 				IXL_SW_RES_SIZE,
5083 				NULL);
5084 	if (error) {
5085 		device_printf(dev,
5086 		    "%s: get_switch_resource_alloc() error %d, aq error %d\n",
5087 		    __func__, error, hw->aq.asq_last_status);
5088 		sbuf_delete(buf);
5089 		return error;
5090 	}
5091 
5092 	/* Sort entries by type for display */
5093 	qsort(resp, num_entries,
5094 	    sizeof(struct i40e_aqc_switch_resource_alloc_element_resp),
5095 	    &ixl_res_alloc_cmp);
5096 
5097 	sbuf_cat(buf, "\n");
5098 	sbuf_printf(buf, "# of entries: %d\n", num_entries);
5099 	sbuf_printf(buf,
5100 	    "Type | Guaranteed | Total | Used   | Un-allocated\n"
5101 	    "     | (this)     | (all) | (this) | (all)       \n");
5102 	for (int i = 0; i < num_entries; i++) {
5103 		sbuf_printf(buf,
5104 		    "%#4x | %10d   %5d   %6d   %12d",
5105 		    resp[i].resource_type,
5106 		    resp[i].guaranteed,
5107 		    resp[i].total,
5108 		    resp[i].used,
5109 		    resp[i].total_unalloced);
5110 		if (i < num_entries - 1)
5111 			sbuf_cat(buf, "\n");
5112 	}
5113 
5114 	error = sbuf_finish(buf);
5115 	if (error) {
5116 		device_printf(dev, "Error finishing sbuf: %d\n", error);
5117 		sbuf_delete(buf);
5118 		return error;
5119 	}
5120 
5121 	error = sysctl_handle_string(oidp, sbuf_data(buf), sbuf_len(buf), req);
5122 	if (error)
5123 		device_printf(dev, "sysctl error: %d\n", error);
5124 	sbuf_delete(buf);
5125 	return error;
5126 }
5127 
5128 /*
5129 ** Caller must init and delete sbuf; this function will clear and
5130 ** finish it for caller.
5131 */
5132 static char *
5133 ixl_switch_element_string(struct sbuf *s, u16 seid, bool uplink)
5134 {
5135 	sbuf_clear(s);
5136 
5137 	if (seid == 0 && uplink)
5138 		sbuf_cat(s, "Network");
5139 	else if (seid == 0)
5140 		sbuf_cat(s, "Host");
5141 	else if (seid == 1)
5142 		sbuf_cat(s, "EMP");
5143 	else if (seid <= 5)
5144 		sbuf_printf(s, "MAC %d", seid - 2);
5145 	else if (seid <= 15)
5146 		sbuf_cat(s, "Reserved");
5147 	else if (seid <= 31)
5148 		sbuf_printf(s, "PF %d", seid - 16);
5149 	else if (seid <= 159)
5150 		sbuf_printf(s, "VF %d", seid - 32);
5151 	else if (seid <= 287)
5152 		sbuf_cat(s, "Reserved");
5153 	else if (seid <= 511)
5154 		sbuf_cat(s, "Other"); // for other structures
5155 	else if (seid <= 895)
5156 		sbuf_printf(s, "VSI %d", seid - 512);
5157 	else if (seid <= 1023)
5158 		sbuf_printf(s, "Reserved");
5159 	else
5160 		sbuf_cat(s, "Invalid");
5161 
5162 	sbuf_finish(s);
5163 	return sbuf_data(s);
5164 }
5165 
5166 static int
5167 ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS)
5168 {
5169 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
5170 	struct i40e_hw *hw = &pf->hw;
5171 	device_t dev = pf->dev;
5172 	struct sbuf *buf;
5173 	struct sbuf *nmbuf;
5174 	int error = 0;
5175 	u8 aq_buf[I40E_AQ_LARGE_BUF];
5176 
5177 	u16 next = 0;
5178 	struct i40e_aqc_get_switch_config_resp *sw_config;
5179 	sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
5180 
5181 	buf = sbuf_new_for_sysctl(NULL, NULL, 0, req);
5182 	if (!buf) {
5183 		device_printf(dev, "Could not allocate sbuf for sysctl output.\n");
5184 		return (ENOMEM);
5185 	}
5186 
5187 	error = i40e_aq_get_switch_config(hw, sw_config,
5188 	    sizeof(aq_buf), &next, NULL);
5189 	if (error) {
5190 		device_printf(dev,
5191 		    "%s: aq_get_switch_config() error %d, aq error %d\n",
5192 		    __func__, error, hw->aq.asq_last_status);
5193 		sbuf_delete(buf);
5194 		return error;
5195 	}
5196 
5197 	nmbuf = sbuf_new_auto();
5198 	if (!nmbuf) {
5199 		device_printf(dev, "Could not allocate sbuf for name output.\n");
5200 		return (ENOMEM);
5201 	}
5202 
5203 	sbuf_cat(buf, "\n");
5204 	// Assuming <= 255 elements in switch
5205 	sbuf_printf(buf, "# of elements: %d\n", sw_config->header.num_reported);
5206 	/* Exclude:
5207 	** Revision -- all elements are revision 1 for now
5208 	*/
5209 	sbuf_printf(buf,
5210 	    "SEID (  Name  ) |  Uplink  | Downlink | Conn Type\n"
5211 	    "                |          |          | (uplink)\n");
5212 	for (int i = 0; i < sw_config->header.num_reported; i++) {
5213 		// "%4d (%8s) | %8s   %8s   %#8x",
5214 		sbuf_printf(buf, "%4d", sw_config->element[i].seid);
5215 		sbuf_cat(buf, " ");
5216 		sbuf_printf(buf, "(%8s)", ixl_switch_element_string(nmbuf,
5217 		    sw_config->element[i].seid, false));
5218 		sbuf_cat(buf, " | ");
5219 		sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf,
5220 		    sw_config->element[i].uplink_seid, true));
5221 		sbuf_cat(buf, "   ");
5222 		sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf,
5223 		    sw_config->element[i].downlink_seid, false));
5224 		sbuf_cat(buf, "   ");
5225 		sbuf_printf(buf, "%#8x", sw_config->element[i].connection_type);
5226 		if (i < sw_config->header.num_reported - 1)
5227 			sbuf_cat(buf, "\n");
5228 	}
5229 	sbuf_delete(nmbuf);
5230 
5231 	error = sbuf_finish(buf);
5232 	if (error) {
5233 		device_printf(dev, "Error finishing sbuf: %d\n", error);
5234 		sbuf_delete(buf);
5235 		return error;
5236 	}
5237 
5238 	error = sysctl_handle_string(oidp, sbuf_data(buf), sbuf_len(buf), req);
5239 	if (error)
5240 		device_printf(dev, "sysctl error: %d\n", error);
5241 	sbuf_delete(buf);
5242 
5243 	return (error);
5244 }
5245 #endif /* IXL_DEBUG_SYSCTL */
5246 
5247 
5248 #ifdef PCI_IOV
5249 static int
5250 ixl_vf_alloc_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
5251 {
5252 	struct i40e_hw *hw;
5253 	struct ixl_vsi *vsi;
5254 	struct i40e_vsi_context vsi_ctx;
5255 	int i;
5256 	uint16_t first_queue;
5257 	enum i40e_status_code code;
5258 
5259 	hw = &pf->hw;
5260 	vsi = &pf->vsi;
5261 
5262 	vsi_ctx.pf_num = hw->pf_id;
5263 	vsi_ctx.uplink_seid = pf->veb_seid;
5264 	vsi_ctx.connection_type = IXL_VSI_DATA_PORT;
5265 	vsi_ctx.vf_num = hw->func_caps.vf_base_id + vf->vf_num;
5266 	vsi_ctx.flags = I40E_AQ_VSI_TYPE_VF;
5267 
5268 	bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
5269 
5270 	vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_SWITCH_VALID);
5271 	vsi_ctx.info.switch_id = htole16(0);
5272 
5273 	vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_SECURITY_VALID);
5274 	vsi_ctx.info.sec_flags = 0;
5275 	if (vf->vf_flags & VF_FLAG_MAC_ANTI_SPOOF)
5276 		vsi_ctx.info.sec_flags |= I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK;
5277 
5278 	vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
5279 	vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
5280 	    I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
5281 
5282 	vsi_ctx.info.valid_sections |=
5283 	    htole16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
5284 	vsi_ctx.info.mapping_flags = htole16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
5285 	first_queue = vsi->num_queues + vf->vf_num * IXLV_MAX_QUEUES;
5286 	for (i = 0; i < IXLV_MAX_QUEUES; i++)
5287 		vsi_ctx.info.queue_mapping[i] = htole16(first_queue + i);
5288 	for (; i < nitems(vsi_ctx.info.queue_mapping); i++)
5289 		vsi_ctx.info.queue_mapping[i] = htole16(I40E_AQ_VSI_QUEUE_MASK);
5290 
5291 	vsi_ctx.info.tc_mapping[0] = htole16(
5292 	    (0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
5293 	    (1 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
5294 
5295 	code = i40e_aq_add_vsi(hw, &vsi_ctx, NULL);
5296 	if (code != I40E_SUCCESS)
5297 		return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5298 	vf->vsi.seid = vsi_ctx.seid;
5299 	vf->vsi.vsi_num = vsi_ctx.vsi_number;
5300 	vf->vsi.first_queue = first_queue;
5301 	vf->vsi.num_queues = IXLV_MAX_QUEUES;
5302 
5303 	code = i40e_aq_get_vsi_params(hw, &vsi_ctx, NULL);
5304 	if (code != I40E_SUCCESS)
5305 		return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5306 
5307 	code = i40e_aq_config_vsi_bw_limit(hw, vf->vsi.seid, 0, 0, NULL);
5308 	if (code != I40E_SUCCESS) {
5309 		device_printf(pf->dev, "Failed to disable BW limit: %d\n",
5310 		    ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5311 		return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5312 	}
5313 
5314 	memcpy(&vf->vsi.info, &vsi_ctx.info, sizeof(vf->vsi.info));
5315 	return (0);
5316 }
5317 
5318 static int
5319 ixl_vf_setup_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
5320 {
5321 	struct i40e_hw *hw;
5322 	int error;
5323 
5324 	hw = &pf->hw;
5325 
5326 	error = ixl_vf_alloc_vsi(pf, vf);
5327 	if (error != 0)
5328 		return (error);
5329 
5330 	vf->vsi.hw_filters_add = 0;
5331 	vf->vsi.hw_filters_del = 0;
5332 	ixl_add_filter(&vf->vsi, ixl_bcast_addr, IXL_VLAN_ANY);
5333 	ixl_reconfigure_filters(&vf->vsi);
5334 
5335 	return (0);
5336 }
5337 
5338 static void
5339 ixl_vf_map_vsi_queue(struct i40e_hw *hw, struct ixl_vf *vf, int qnum,
5340     uint32_t val)
5341 {
5342 	uint32_t qtable;
5343 	int index, shift;
5344 
5345 	/*
5346 	 * Two queues are mapped in a single register, so we have to do some
5347 	 * gymnastics to convert the queue number into a register index and
5348 	 * shift.
5349 	 */
5350 	index = qnum / 2;
5351 	shift = (qnum % 2) * I40E_VSILAN_QTABLE_QINDEX_1_SHIFT;
5352 
5353 	qtable = rd32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num));
5354 	qtable &= ~(I40E_VSILAN_QTABLE_QINDEX_0_MASK << shift);
5355 	qtable |= val << shift;
5356 	wr32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num), qtable);
5357 }
5358 
5359 static void
5360 ixl_vf_map_queues(struct ixl_pf *pf, struct ixl_vf *vf)
5361 {
5362 	struct i40e_hw *hw;
5363 	uint32_t qtable;
5364 	int i;
5365 
5366 	hw = &pf->hw;
5367 
5368 	/*
5369 	 * Contiguous mappings aren't actually supported by the hardware,
5370 	 * so we have to use non-contiguous mappings.
5371 	 */
5372 	wr32(hw, I40E_VSILAN_QBASE(vf->vsi.vsi_num),
5373 	     I40E_VSILAN_QBASE_VSIQTABLE_ENA_MASK);
5374 
5375 	wr32(hw, I40E_VPLAN_MAPENA(vf->vf_num),
5376 	    I40E_VPLAN_MAPENA_TXRX_ENA_MASK);
5377 
5378 	for (i = 0; i < vf->vsi.num_queues; i++) {
5379 		qtable = (vf->vsi.first_queue + i) <<
5380 		    I40E_VPLAN_QTABLE_QINDEX_SHIFT;
5381 
5382 		wr32(hw, I40E_VPLAN_QTABLE(i, vf->vf_num), qtable);
5383 	}
5384 
5385 	/* Map queues allocated to VF to its VSI. */
5386 	for (i = 0; i < vf->vsi.num_queues; i++)
5387 		ixl_vf_map_vsi_queue(hw, vf, i, vf->vsi.first_queue + i);
5388 
5389 	/* Set rest of VSI queues as unused. */
5390 	for (; i < IXL_MAX_VSI_QUEUES; i++)
5391 		ixl_vf_map_vsi_queue(hw, vf, i,
5392 		    I40E_VSILAN_QTABLE_QINDEX_0_MASK);
5393 
5394 	ixl_flush(hw);
5395 }
5396 
5397 static void
5398 ixl_vf_vsi_release(struct ixl_pf *pf, struct ixl_vsi *vsi)
5399 {
5400 	struct i40e_hw *hw;
5401 
5402 	hw = &pf->hw;
5403 
5404 	if (vsi->seid == 0)
5405 		return;
5406 
5407 	i40e_aq_delete_element(hw, vsi->seid, NULL);
5408 }
5409 
5410 static void
5411 ixl_vf_disable_queue_intr(struct i40e_hw *hw, uint32_t vfint_reg)
5412 {
5413 
5414 	wr32(hw, vfint_reg, I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
5415 	ixl_flush(hw);
5416 }
5417 
5418 static void
5419 ixl_vf_unregister_intr(struct i40e_hw *hw, uint32_t vpint_reg)
5420 {
5421 
5422 	wr32(hw, vpint_reg, I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_MASK |
5423 	    I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK);
5424 	ixl_flush(hw);
5425 }
5426 
5427 static void
5428 ixl_vf_release_resources(struct ixl_pf *pf, struct ixl_vf *vf)
5429 {
5430 	struct i40e_hw *hw;
5431 	uint32_t vfint_reg, vpint_reg;
5432 	int i;
5433 
5434 	hw = &pf->hw;
5435 
5436 	ixl_vf_vsi_release(pf, &vf->vsi);
5437 
5438 	/* Index 0 has a special register. */
5439 	ixl_vf_disable_queue_intr(hw, I40E_VFINT_DYN_CTL0(vf->vf_num));
5440 
5441 	for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
5442 		vfint_reg = IXL_VFINT_DYN_CTLN_REG(hw, i , vf->vf_num);
5443 		ixl_vf_disable_queue_intr(hw, vfint_reg);
5444 	}
5445 
5446 	/* Index 0 has a special register. */
5447 	ixl_vf_unregister_intr(hw, I40E_VPINT_LNKLST0(vf->vf_num));
5448 
5449 	for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
5450 		vpint_reg = IXL_VPINT_LNKLSTN_REG(hw, i, vf->vf_num);
5451 		ixl_vf_unregister_intr(hw, vpint_reg);
5452 	}
5453 
5454 	vf->vsi.num_queues = 0;
5455 }
5456 
5457 static int
5458 ixl_flush_pcie(struct ixl_pf *pf, struct ixl_vf *vf)
5459 {
5460 	struct i40e_hw *hw;
5461 	int i;
5462 	uint16_t global_vf_num;
5463 	uint32_t ciad;
5464 
5465 	hw = &pf->hw;
5466 	global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
5467 
5468 	wr32(hw, I40E_PF_PCI_CIAA, IXL_PF_PCI_CIAA_VF_DEVICE_STATUS |
5469 	     (global_vf_num << I40E_PF_PCI_CIAA_VF_NUM_SHIFT));
5470 	for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
5471 		ciad = rd32(hw, I40E_PF_PCI_CIAD);
5472 		if ((ciad & IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK) == 0)
5473 			return (0);
5474 		DELAY(1);
5475 	}
5476 
5477 	return (ETIMEDOUT);
5478 }
5479 
5480 static void
5481 ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf)
5482 {
5483 	struct i40e_hw *hw;
5484 	uint32_t vfrtrig;
5485 
5486 	hw = &pf->hw;
5487 
5488 	vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
5489 	vfrtrig |= I40E_VPGEN_VFRTRIG_VFSWR_MASK;
5490 	wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
5491 	ixl_flush(hw);
5492 
5493 	ixl_reinit_vf(pf, vf);
5494 }
5495 
5496 static void
5497 ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf)
5498 {
5499 	struct i40e_hw *hw;
5500 	uint32_t vfrstat, vfrtrig;
5501 	int i, error;
5502 
5503 	hw = &pf->hw;
5504 
5505 	error = ixl_flush_pcie(pf, vf);
5506 	if (error != 0)
5507 		device_printf(pf->dev,
5508 		    "Timed out waiting for PCIe activity to stop on VF-%d\n",
5509 		    vf->vf_num);
5510 
5511 	for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
5512 		DELAY(10);
5513 
5514 		vfrstat = rd32(hw, I40E_VPGEN_VFRSTAT(vf->vf_num));
5515 		if (vfrstat & I40E_VPGEN_VFRSTAT_VFRD_MASK)
5516 			break;
5517 	}
5518 
5519 	if (i == IXL_VF_RESET_TIMEOUT)
5520 		device_printf(pf->dev, "VF %d failed to reset\n", vf->vf_num);
5521 
5522 	wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_COMPLETED);
5523 
5524 	vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
5525 	vfrtrig &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK;
5526 	wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
5527 
5528 	if (vf->vsi.seid != 0)
5529 		ixl_disable_rings(&vf->vsi);
5530 
5531 	ixl_vf_release_resources(pf, vf);
5532 	ixl_vf_setup_vsi(pf, vf);
5533 	ixl_vf_map_queues(pf, vf);
5534 
5535 	wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_VFACTIVE);
5536 	ixl_flush(hw);
5537 }
5538 
5539 static const char *
5540 ixl_vc_opcode_str(uint16_t op)
5541 {
5542 
5543 	switch (op) {
5544 	case I40E_VIRTCHNL_OP_VERSION:
5545 		return ("VERSION");
5546 	case I40E_VIRTCHNL_OP_RESET_VF:
5547 		return ("RESET_VF");
5548 	case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
5549 		return ("GET_VF_RESOURCES");
5550 	case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
5551 		return ("CONFIG_TX_QUEUE");
5552 	case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
5553 		return ("CONFIG_RX_QUEUE");
5554 	case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
5555 		return ("CONFIG_VSI_QUEUES");
5556 	case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
5557 		return ("CONFIG_IRQ_MAP");
5558 	case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
5559 		return ("ENABLE_QUEUES");
5560 	case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
5561 		return ("DISABLE_QUEUES");
5562 	case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
5563 		return ("ADD_ETHER_ADDRESS");
5564 	case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
5565 		return ("DEL_ETHER_ADDRESS");
5566 	case I40E_VIRTCHNL_OP_ADD_VLAN:
5567 		return ("ADD_VLAN");
5568 	case I40E_VIRTCHNL_OP_DEL_VLAN:
5569 		return ("DEL_VLAN");
5570 	case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
5571 		return ("CONFIG_PROMISCUOUS_MODE");
5572 	case I40E_VIRTCHNL_OP_GET_STATS:
5573 		return ("GET_STATS");
5574 	case I40E_VIRTCHNL_OP_FCOE:
5575 		return ("FCOE");
5576 	case I40E_VIRTCHNL_OP_EVENT:
5577 		return ("EVENT");
5578 	default:
5579 		return ("UNKNOWN");
5580 	}
5581 }
5582 
5583 static int
5584 ixl_vc_opcode_level(uint16_t opcode)
5585 {
5586 
5587 	switch (opcode) {
5588 	case I40E_VIRTCHNL_OP_GET_STATS:
5589 		return (10);
5590 	default:
5591 		return (5);
5592 	}
5593 }
5594 
5595 static void
5596 ixl_send_vf_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
5597     enum i40e_status_code status, void *msg, uint16_t len)
5598 {
5599 	struct i40e_hw *hw;
5600 	int global_vf_id;
5601 
5602 	hw = &pf->hw;
5603 	global_vf_id = hw->func_caps.vf_base_id + vf->vf_num;
5604 
5605 	I40E_VC_DEBUG(pf, ixl_vc_opcode_level(op),
5606 	    "Sending msg (op=%s[%d], status=%d) to VF-%d\n",
5607 	    ixl_vc_opcode_str(op), op, status, vf->vf_num);
5608 
5609 	i40e_aq_send_msg_to_vf(hw, global_vf_id, op, status, msg, len, NULL);
5610 }
5611 
5612 static void
5613 ixl_send_vf_ack(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op)
5614 {
5615 
5616 	ixl_send_vf_msg(pf, vf, op, I40E_SUCCESS, NULL, 0);
5617 }
5618 
5619 static void
5620 ixl_send_vf_nack_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
5621     enum i40e_status_code status, const char *file, int line)
5622 {
5623 
5624 	I40E_VC_DEBUG(pf, 1,
5625 	    "Sending NACK (op=%s[%d], err=%d) to VF-%d from %s:%d\n",
5626 	    ixl_vc_opcode_str(op), op, status, vf->vf_num, file, line);
5627 	ixl_send_vf_msg(pf, vf, op, status, NULL, 0);
5628 }
5629 
5630 static void
5631 ixl_vf_version_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5632     uint16_t msg_size)
5633 {
5634 	struct i40e_virtchnl_version_info reply;
5635 
5636 	if (msg_size != sizeof(struct i40e_virtchnl_version_info)) {
5637 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_VERSION,
5638 		    I40E_ERR_PARAM);
5639 		return;
5640 	}
5641 
5642 	reply.major = I40E_VIRTCHNL_VERSION_MAJOR;
5643 	reply.minor = I40E_VIRTCHNL_VERSION_MINOR;
5644 	ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_VERSION, I40E_SUCCESS, &reply,
5645 	    sizeof(reply));
5646 }
5647 
5648 static void
5649 ixl_vf_reset_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5650     uint16_t msg_size)
5651 {
5652 
5653 	if (msg_size != 0) {
5654 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_RESET_VF,
5655 		    I40E_ERR_PARAM);
5656 		return;
5657 	}
5658 
5659 	ixl_reset_vf(pf, vf);
5660 
5661 	/* No response to a reset message. */
5662 }
5663 
5664 static void
5665 ixl_vf_get_resources_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5666     uint16_t msg_size)
5667 {
5668 	struct i40e_virtchnl_vf_resource reply;
5669 
5670 	if (msg_size != 0) {
5671 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
5672 		    I40E_ERR_PARAM);
5673 		return;
5674 	}
5675 
5676 	bzero(&reply, sizeof(reply));
5677 
5678 	reply.vf_offload_flags = I40E_VIRTCHNL_VF_OFFLOAD_L2;
5679 
5680 	reply.num_vsis = 1;
5681 	reply.num_queue_pairs = vf->vsi.num_queues;
5682 	reply.max_vectors = pf->hw.func_caps.num_msix_vectors_vf;
5683 	reply.vsi_res[0].vsi_id = vf->vsi.vsi_num;
5684 	reply.vsi_res[0].vsi_type = I40E_VSI_SRIOV;
5685 	reply.vsi_res[0].num_queue_pairs = vf->vsi.num_queues;
5686 	memcpy(reply.vsi_res[0].default_mac_addr, vf->mac, ETHER_ADDR_LEN);
5687 
5688 	ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
5689 	    I40E_SUCCESS, &reply, sizeof(reply));
5690 }
5691 
5692 static int
5693 ixl_vf_config_tx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
5694     struct i40e_virtchnl_txq_info *info)
5695 {
5696 	struct i40e_hw *hw;
5697 	struct i40e_hmc_obj_txq txq;
5698 	uint16_t global_queue_num, global_vf_num;
5699 	enum i40e_status_code status;
5700 	uint32_t qtx_ctl;
5701 
5702 	hw = &pf->hw;
5703 	global_queue_num = vf->vsi.first_queue + info->queue_id;
5704 	global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
5705 	bzero(&txq, sizeof(txq));
5706 
5707 	status = i40e_clear_lan_tx_queue_context(hw, global_queue_num);
5708 	if (status != I40E_SUCCESS)
5709 		return (EINVAL);
5710 
5711 	txq.base = info->dma_ring_addr / IXL_TX_CTX_BASE_UNITS;
5712 
5713 	txq.head_wb_ena = info->headwb_enabled;
5714 	txq.head_wb_addr = info->dma_headwb_addr;
5715 	txq.qlen = info->ring_len;
5716 	txq.rdylist = le16_to_cpu(vf->vsi.info.qs_handle[0]);
5717 	txq.rdylist_act = 0;
5718 
5719 	status = i40e_set_lan_tx_queue_context(hw, global_queue_num, &txq);
5720 	if (status != I40E_SUCCESS)
5721 		return (EINVAL);
5722 
5723 	qtx_ctl = I40E_QTX_CTL_VF_QUEUE |
5724 	    (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) |
5725 	    (global_vf_num << I40E_QTX_CTL_VFVM_INDX_SHIFT);
5726 	wr32(hw, I40E_QTX_CTL(global_queue_num), qtx_ctl);
5727 	ixl_flush(hw);
5728 
5729 	return (0);
5730 }
5731 
5732 static int
5733 ixl_vf_config_rx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
5734     struct i40e_virtchnl_rxq_info *info)
5735 {
5736 	struct i40e_hw *hw;
5737 	struct i40e_hmc_obj_rxq rxq;
5738 	uint16_t global_queue_num;
5739 	enum i40e_status_code status;
5740 
5741 	hw = &pf->hw;
5742 	global_queue_num = vf->vsi.first_queue + info->queue_id;
5743 	bzero(&rxq, sizeof(rxq));
5744 
5745 	if (info->databuffer_size > IXL_VF_MAX_BUFFER)
5746 		return (EINVAL);
5747 
5748 	if (info->max_pkt_size > IXL_VF_MAX_FRAME ||
5749 	    info->max_pkt_size < ETHER_MIN_LEN)
5750 		return (EINVAL);
5751 
5752 	if (info->splithdr_enabled) {
5753 		if (info->hdr_size > IXL_VF_MAX_HDR_BUFFER)
5754 			return (EINVAL);
5755 
5756 		rxq.hsplit_0 = info->rx_split_pos &
5757 		    (I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_L2 |
5758 		     I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_IP |
5759 		     I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_TCP_UDP |
5760 		     I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_SCTP);
5761 		rxq.hbuff = info->hdr_size >> I40E_RXQ_CTX_HBUFF_SHIFT;
5762 
5763 		rxq.dtype = 2;
5764 	}
5765 
5766 	status = i40e_clear_lan_rx_queue_context(hw, global_queue_num);
5767 	if (status != I40E_SUCCESS)
5768 		return (EINVAL);
5769 
5770 	rxq.base = info->dma_ring_addr / IXL_RX_CTX_BASE_UNITS;
5771 	rxq.qlen = info->ring_len;
5772 
5773 	rxq.dbuff = info->databuffer_size >> I40E_RXQ_CTX_DBUFF_SHIFT;
5774 
5775 	rxq.dsize = 1;
5776 	rxq.crcstrip = 1;
5777 	rxq.l2tsel = 1;
5778 
5779 	rxq.rxmax = info->max_pkt_size;
5780 	rxq.tphrdesc_ena = 1;
5781 	rxq.tphwdesc_ena = 1;
5782 	rxq.tphdata_ena = 1;
5783 	rxq.tphhead_ena = 1;
5784 	rxq.lrxqthresh = 2;
5785 	rxq.prefena = 1;
5786 
5787 	status = i40e_set_lan_rx_queue_context(hw, global_queue_num, &rxq);
5788 	if (status != I40E_SUCCESS)
5789 		return (EINVAL);
5790 
5791 	return (0);
5792 }
5793 
5794 static void
5795 ixl_vf_config_vsi_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5796     uint16_t msg_size)
5797 {
5798 	struct i40e_virtchnl_vsi_queue_config_info *info;
5799 	struct i40e_virtchnl_queue_pair_info *pair;
5800 	int i;
5801 
5802 	if (msg_size < sizeof(*info)) {
5803 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5804 		    I40E_ERR_PARAM);
5805 		return;
5806 	}
5807 
5808 	info = msg;
5809 	if (info->num_queue_pairs == 0) {
5810 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5811 		    I40E_ERR_PARAM);
5812 		return;
5813 	}
5814 
5815 	if (msg_size != sizeof(*info) + info->num_queue_pairs * sizeof(*pair)) {
5816 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5817 		    I40E_ERR_PARAM);
5818 		return;
5819 	}
5820 
5821 	if (info->vsi_id != vf->vsi.vsi_num) {
5822 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5823 		    I40E_ERR_PARAM);
5824 		return;
5825 	}
5826 
5827 	for (i = 0; i < info->num_queue_pairs; i++) {
5828 		pair = &info->qpair[i];
5829 
5830 		if (pair->txq.vsi_id != vf->vsi.vsi_num ||
5831 		    pair->rxq.vsi_id != vf->vsi.vsi_num ||
5832 		    pair->txq.queue_id != pair->rxq.queue_id ||
5833 		    pair->txq.queue_id >= vf->vsi.num_queues) {
5834 
5835 			i40e_send_vf_nack(pf, vf,
5836 			    I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
5837 			return;
5838 		}
5839 
5840 		if (ixl_vf_config_tx_queue(pf, vf, &pair->txq) != 0) {
5841 			i40e_send_vf_nack(pf, vf,
5842 			    I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
5843 			return;
5844 		}
5845 
5846 		if (ixl_vf_config_rx_queue(pf, vf, &pair->rxq) != 0) {
5847 			i40e_send_vf_nack(pf, vf,
5848 			    I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
5849 			return;
5850 		}
5851 	}
5852 
5853 	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES);
5854 }
5855 
5856 static void
5857 ixl_vf_set_qctl(struct ixl_pf *pf,
5858     const struct i40e_virtchnl_vector_map *vector,
5859     enum i40e_queue_type cur_type, uint16_t cur_queue,
5860     enum i40e_queue_type *last_type, uint16_t *last_queue)
5861 {
5862 	uint32_t offset, qctl;
5863 	uint16_t itr_indx;
5864 
5865 	if (cur_type == I40E_QUEUE_TYPE_RX) {
5866 		offset = I40E_QINT_RQCTL(cur_queue);
5867 		itr_indx = vector->rxitr_idx;
5868 	} else {
5869 		offset = I40E_QINT_TQCTL(cur_queue);
5870 		itr_indx = vector->txitr_idx;
5871 	}
5872 
5873 	qctl = htole32((vector->vector_id << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
5874 	    (*last_type << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
5875 	    (*last_queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
5876 	    I40E_QINT_RQCTL_CAUSE_ENA_MASK |
5877 	    (itr_indx << I40E_QINT_RQCTL_ITR_INDX_SHIFT));
5878 
5879 	wr32(&pf->hw, offset, qctl);
5880 
5881 	*last_type = cur_type;
5882 	*last_queue = cur_queue;
5883 }
5884 
5885 static void
5886 ixl_vf_config_vector(struct ixl_pf *pf, struct ixl_vf *vf,
5887     const struct i40e_virtchnl_vector_map *vector)
5888 {
5889 	struct i40e_hw *hw;
5890 	u_int qindex;
5891 	enum i40e_queue_type type, last_type;
5892 	uint32_t lnklst_reg;
5893 	uint16_t rxq_map, txq_map, cur_queue, last_queue;
5894 
5895 	hw = &pf->hw;
5896 
5897 	rxq_map = vector->rxq_map;
5898 	txq_map = vector->txq_map;
5899 
5900 	last_queue = IXL_END_OF_INTR_LNKLST;
5901 	last_type = I40E_QUEUE_TYPE_RX;
5902 
5903 	/*
5904 	 * The datasheet says to optimize performance, RX queues and TX queues
5905 	 * should be interleaved in the interrupt linked list, so we process
5906 	 * both at once here.
5907 	 */
5908 	while ((rxq_map != 0) || (txq_map != 0)) {
5909 		if (txq_map != 0) {
5910 			qindex = ffs(txq_map) - 1;
5911 			type = I40E_QUEUE_TYPE_TX;
5912 			cur_queue = vf->vsi.first_queue + qindex;
5913 			ixl_vf_set_qctl(pf, vector, type, cur_queue,
5914 			    &last_type, &last_queue);
5915 			txq_map &= ~(1 << qindex);
5916 		}
5917 
5918 		if (rxq_map != 0) {
5919 			qindex = ffs(rxq_map) - 1;
5920 			type = I40E_QUEUE_TYPE_RX;
5921 			cur_queue = vf->vsi.first_queue + qindex;
5922 			ixl_vf_set_qctl(pf, vector, type, cur_queue,
5923 			    &last_type, &last_queue);
5924 			rxq_map &= ~(1 << qindex);
5925 		}
5926 	}
5927 
5928 	if (vector->vector_id == 0)
5929 		lnklst_reg = I40E_VPINT_LNKLST0(vf->vf_num);
5930 	else
5931 		lnklst_reg = IXL_VPINT_LNKLSTN_REG(hw, vector->vector_id,
5932 		    vf->vf_num);
5933 	wr32(hw, lnklst_reg,
5934 	    (last_queue << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
5935 	    (last_type << I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
5936 
5937 	ixl_flush(hw);
5938 }
5939 
5940 static void
5941 ixl_vf_config_irq_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5942     uint16_t msg_size)
5943 {
5944 	struct i40e_virtchnl_irq_map_info *map;
5945 	struct i40e_virtchnl_vector_map *vector;
5946 	struct i40e_hw *hw;
5947 	int i, largest_txq, largest_rxq;
5948 
5949 	hw = &pf->hw;
5950 
5951 	if (msg_size < sizeof(*map)) {
5952 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5953 		    I40E_ERR_PARAM);
5954 		return;
5955 	}
5956 
5957 	map = msg;
5958 	if (map->num_vectors == 0) {
5959 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5960 		    I40E_ERR_PARAM);
5961 		return;
5962 	}
5963 
5964 	if (msg_size != sizeof(*map) + map->num_vectors * sizeof(*vector)) {
5965 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5966 		    I40E_ERR_PARAM);
5967 		return;
5968 	}
5969 
5970 	for (i = 0; i < map->num_vectors; i++) {
5971 		vector = &map->vecmap[i];
5972 
5973 		if ((vector->vector_id >= hw->func_caps.num_msix_vectors_vf) ||
5974 		    vector->vsi_id != vf->vsi.vsi_num) {
5975 			i40e_send_vf_nack(pf, vf,
5976 			    I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_ERR_PARAM);
5977 			return;
5978 		}
5979 
5980 		if (vector->rxq_map != 0) {
5981 			largest_rxq = fls(vector->rxq_map) - 1;
5982 			if (largest_rxq >= vf->vsi.num_queues) {
5983 				i40e_send_vf_nack(pf, vf,
5984 				    I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5985 				    I40E_ERR_PARAM);
5986 				return;
5987 			}
5988 		}
5989 
5990 		if (vector->txq_map != 0) {
5991 			largest_txq = fls(vector->txq_map) - 1;
5992 			if (largest_txq >= vf->vsi.num_queues) {
5993 				i40e_send_vf_nack(pf, vf,
5994 				    I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5995 				    I40E_ERR_PARAM);
5996 				return;
5997 			}
5998 		}
5999 
6000 		if (vector->rxitr_idx > IXL_MAX_ITR_IDX ||
6001 		    vector->txitr_idx > IXL_MAX_ITR_IDX) {
6002 			i40e_send_vf_nack(pf, vf,
6003 			    I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6004 			    I40E_ERR_PARAM);
6005 			return;
6006 		}
6007 
6008 		ixl_vf_config_vector(pf, vf, vector);
6009 	}
6010 
6011 	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP);
6012 }
6013 
6014 static void
6015 ixl_vf_enable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6016     uint16_t msg_size)
6017 {
6018 	struct i40e_virtchnl_queue_select *select;
6019 	int error;
6020 
6021 	if (msg_size != sizeof(*select)) {
6022 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6023 		    I40E_ERR_PARAM);
6024 		return;
6025 	}
6026 
6027 	select = msg;
6028 	if (select->vsi_id != vf->vsi.vsi_num ||
6029 	    select->rx_queues == 0 || select->tx_queues == 0) {
6030 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6031 		    I40E_ERR_PARAM);
6032 		return;
6033 	}
6034 
6035 	error = ixl_enable_rings(&vf->vsi);
6036 	if (error) {
6037 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6038 		    I40E_ERR_TIMEOUT);
6039 		return;
6040 	}
6041 
6042 	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES);
6043 }
6044 
6045 static void
6046 ixl_vf_disable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf,
6047     void *msg, uint16_t msg_size)
6048 {
6049 	struct i40e_virtchnl_queue_select *select;
6050 	int error;
6051 
6052 	if (msg_size != sizeof(*select)) {
6053 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6054 		    I40E_ERR_PARAM);
6055 		return;
6056 	}
6057 
6058 	select = msg;
6059 	if (select->vsi_id != vf->vsi.vsi_num ||
6060 	    select->rx_queues == 0 || select->tx_queues == 0) {
6061 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6062 		    I40E_ERR_PARAM);
6063 		return;
6064 	}
6065 
6066 	error = ixl_disable_rings(&vf->vsi);
6067 	if (error) {
6068 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6069 		    I40E_ERR_TIMEOUT);
6070 		return;
6071 	}
6072 
6073 	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES);
6074 }
6075 
6076 static boolean_t
6077 ixl_zero_mac(const uint8_t *addr)
6078 {
6079 	uint8_t zero[ETHER_ADDR_LEN] = {0, 0, 0, 0, 0, 0};
6080 
6081 	return (cmp_etheraddr(addr, zero));
6082 }
6083 
6084 static boolean_t
6085 ixl_bcast_mac(const uint8_t *addr)
6086 {
6087 
6088 	return (cmp_etheraddr(addr, ixl_bcast_addr));
6089 }
6090 
6091 static int
6092 ixl_vf_mac_valid(struct ixl_vf *vf, const uint8_t *addr)
6093 {
6094 
6095 	if (ixl_zero_mac(addr) || ixl_bcast_mac(addr))
6096 		return (EINVAL);
6097 
6098 	/*
6099 	 * If the VF is not allowed to change its MAC address, don't let it
6100 	 * set a MAC filter for an address that is not a multicast address and
6101 	 * is not its assigned MAC.
6102 	 */
6103 	if (!(vf->vf_flags & VF_FLAG_SET_MAC_CAP) &&
6104 	    !(ETHER_IS_MULTICAST(addr) || cmp_etheraddr(addr, vf->mac)))
6105 		return (EPERM);
6106 
6107 	return (0);
6108 }
6109 
6110 static void
6111 ixl_vf_add_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6112     uint16_t msg_size)
6113 {
6114 	struct i40e_virtchnl_ether_addr_list *addr_list;
6115 	struct i40e_virtchnl_ether_addr *addr;
6116 	struct ixl_vsi *vsi;
6117 	int i;
6118 	size_t expected_size;
6119 
6120 	vsi = &vf->vsi;
6121 
6122 	if (msg_size < sizeof(*addr_list)) {
6123 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6124 		    I40E_ERR_PARAM);
6125 		return;
6126 	}
6127 
6128 	addr_list = msg;
6129 	expected_size = sizeof(*addr_list) +
6130 	    addr_list->num_elements * sizeof(*addr);
6131 
6132 	if (addr_list->num_elements == 0 ||
6133 	    addr_list->vsi_id != vsi->vsi_num ||
6134 	    msg_size != expected_size) {
6135 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6136 		    I40E_ERR_PARAM);
6137 		return;
6138 	}
6139 
6140 	for (i = 0; i < addr_list->num_elements; i++) {
6141 		if (ixl_vf_mac_valid(vf, addr_list->list[i].addr) != 0) {
6142 			i40e_send_vf_nack(pf, vf,
6143 			    I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6144 			return;
6145 		}
6146 	}
6147 
6148 	for (i = 0; i < addr_list->num_elements; i++) {
6149 		addr = &addr_list->list[i];
6150 		ixl_add_filter(vsi, addr->addr, IXL_VLAN_ANY);
6151 	}
6152 
6153 	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS);
6154 }
6155 
6156 static void
6157 ixl_vf_del_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6158     uint16_t msg_size)
6159 {
6160 	struct i40e_virtchnl_ether_addr_list *addr_list;
6161 	struct i40e_virtchnl_ether_addr *addr;
6162 	size_t expected_size;
6163 	int i;
6164 
6165 	if (msg_size < sizeof(*addr_list)) {
6166 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6167 		    I40E_ERR_PARAM);
6168 		return;
6169 	}
6170 
6171 	addr_list = msg;
6172 	expected_size = sizeof(*addr_list) +
6173 	    addr_list->num_elements * sizeof(*addr);
6174 
6175 	if (addr_list->num_elements == 0 ||
6176 	    addr_list->vsi_id != vf->vsi.vsi_num ||
6177 	    msg_size != expected_size) {
6178 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6179 		    I40E_ERR_PARAM);
6180 		return;
6181 	}
6182 
6183 	for (i = 0; i < addr_list->num_elements; i++) {
6184 		addr = &addr_list->list[i];
6185 		if (ixl_zero_mac(addr->addr) || ixl_bcast_mac(addr->addr)) {
6186 			i40e_send_vf_nack(pf, vf,
6187 			    I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6188 			return;
6189 		}
6190 	}
6191 
6192 	for (i = 0; i < addr_list->num_elements; i++) {
6193 		addr = &addr_list->list[i];
6194 		ixl_del_filter(&vf->vsi, addr->addr, IXL_VLAN_ANY);
6195 	}
6196 
6197 	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS);
6198 }
6199 
6200 static enum i40e_status_code
6201 ixl_vf_enable_vlan_strip(struct ixl_pf *pf, struct ixl_vf *vf)
6202 {
6203 	struct i40e_vsi_context vsi_ctx;
6204 
6205 	vsi_ctx.seid = vf->vsi.seid;
6206 
6207 	bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
6208 	vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
6209 	vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
6210 	    I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
6211 	return (i40e_aq_update_vsi_params(&pf->hw, &vsi_ctx, NULL));
6212 }
6213 
6214 static void
6215 ixl_vf_add_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6216     uint16_t msg_size)
6217 {
6218 	struct i40e_virtchnl_vlan_filter_list *filter_list;
6219 	enum i40e_status_code code;
6220 	size_t expected_size;
6221 	int i;
6222 
6223 	if (msg_size < sizeof(*filter_list)) {
6224 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6225 		    I40E_ERR_PARAM);
6226 		return;
6227 	}
6228 
6229 	filter_list = msg;
6230 	expected_size = sizeof(*filter_list) +
6231 	    filter_list->num_elements * sizeof(uint16_t);
6232 	if (filter_list->num_elements == 0 ||
6233 	    filter_list->vsi_id != vf->vsi.vsi_num ||
6234 	    msg_size != expected_size) {
6235 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6236 		    I40E_ERR_PARAM);
6237 		return;
6238 	}
6239 
6240 	if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
6241 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6242 		    I40E_ERR_PARAM);
6243 		return;
6244 	}
6245 
6246 	for (i = 0; i < filter_list->num_elements; i++) {
6247 		if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
6248 			i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6249 			    I40E_ERR_PARAM);
6250 			return;
6251 		}
6252 	}
6253 
6254 	code = ixl_vf_enable_vlan_strip(pf, vf);
6255 	if (code != I40E_SUCCESS) {
6256 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6257 		    I40E_ERR_PARAM);
6258 	}
6259 
6260 	for (i = 0; i < filter_list->num_elements; i++)
6261 		ixl_add_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
6262 
6263 	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN);
6264 }
6265 
6266 static void
6267 ixl_vf_del_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6268     uint16_t msg_size)
6269 {
6270 	struct i40e_virtchnl_vlan_filter_list *filter_list;
6271 	int i;
6272 	size_t expected_size;
6273 
6274 	if (msg_size < sizeof(*filter_list)) {
6275 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
6276 		    I40E_ERR_PARAM);
6277 		return;
6278 	}
6279 
6280 	filter_list = msg;
6281 	expected_size = sizeof(*filter_list) +
6282 	    filter_list->num_elements * sizeof(uint16_t);
6283 	if (filter_list->num_elements == 0 ||
6284 	    filter_list->vsi_id != vf->vsi.vsi_num ||
6285 	    msg_size != expected_size) {
6286 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
6287 		    I40E_ERR_PARAM);
6288 		return;
6289 	}
6290 
6291 	for (i = 0; i < filter_list->num_elements; i++) {
6292 		if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
6293 			i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6294 			    I40E_ERR_PARAM);
6295 			return;
6296 		}
6297 	}
6298 
6299 	if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
6300 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6301 		    I40E_ERR_PARAM);
6302 		return;
6303 	}
6304 
6305 	for (i = 0; i < filter_list->num_elements; i++)
6306 		ixl_del_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
6307 
6308 	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN);
6309 }
6310 
6311 static void
6312 ixl_vf_config_promisc_msg(struct ixl_pf *pf, struct ixl_vf *vf,
6313     void *msg, uint16_t msg_size)
6314 {
6315 	struct i40e_virtchnl_promisc_info *info;
6316 	enum i40e_status_code code;
6317 
6318 	if (msg_size != sizeof(*info)) {
6319 		i40e_send_vf_nack(pf, vf,
6320 		    I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
6321 		return;
6322 	}
6323 
6324 	if (!vf->vf_flags & VF_FLAG_PROMISC_CAP) {
6325 		i40e_send_vf_nack(pf, vf,
6326 		    I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
6327 		return;
6328 	}
6329 
6330 	info = msg;
6331 	if (info->vsi_id != vf->vsi.vsi_num) {
6332 		i40e_send_vf_nack(pf, vf,
6333 		    I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
6334 		return;
6335 	}
6336 
6337 	code = i40e_aq_set_vsi_unicast_promiscuous(&pf->hw, info->vsi_id,
6338 	    info->flags & I40E_FLAG_VF_UNICAST_PROMISC, NULL);
6339 	if (code != I40E_SUCCESS) {
6340 		i40e_send_vf_nack(pf, vf,
6341 		    I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
6342 		return;
6343 	}
6344 
6345 	code = i40e_aq_set_vsi_multicast_promiscuous(&pf->hw, info->vsi_id,
6346 	    info->flags & I40E_FLAG_VF_MULTICAST_PROMISC, NULL);
6347 	if (code != I40E_SUCCESS) {
6348 		i40e_send_vf_nack(pf, vf,
6349 		    I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
6350 		return;
6351 	}
6352 
6353 	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE);
6354 }
6355 
6356 static void
6357 ixl_vf_get_stats_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6358     uint16_t msg_size)
6359 {
6360 	struct i40e_virtchnl_queue_select *queue;
6361 
6362 	if (msg_size != sizeof(*queue)) {
6363 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
6364 		    I40E_ERR_PARAM);
6365 		return;
6366 	}
6367 
6368 	queue = msg;
6369 	if (queue->vsi_id != vf->vsi.vsi_num) {
6370 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
6371 		    I40E_ERR_PARAM);
6372 		return;
6373 	}
6374 
6375 	ixl_update_eth_stats(&vf->vsi);
6376 
6377 	ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
6378 	    I40E_SUCCESS, &vf->vsi.eth_stats, sizeof(vf->vsi.eth_stats));
6379 }
6380 
6381 static void
6382 ixl_handle_vf_msg(struct ixl_pf *pf, struct i40e_arq_event_info *event)
6383 {
6384 	struct ixl_vf *vf;
6385 	void *msg;
6386 	uint16_t vf_num, msg_size;
6387 	uint32_t opcode;
6388 
6389 	vf_num = le16toh(event->desc.retval) - pf->hw.func_caps.vf_base_id;
6390 	opcode = le32toh(event->desc.cookie_high);
6391 
6392 	if (vf_num >= pf->num_vfs) {
6393 		device_printf(pf->dev, "Got msg from illegal VF: %d\n", vf_num);
6394 		return;
6395 	}
6396 
6397 	vf = &pf->vfs[vf_num];
6398 	msg = event->msg_buf;
6399 	msg_size = event->msg_len;
6400 
6401 	I40E_VC_DEBUG(pf, ixl_vc_opcode_level(opcode),
6402 	    "Got msg %s(%d) from VF-%d of size %d\n",
6403 	    ixl_vc_opcode_str(opcode), opcode, vf_num, msg_size);
6404 
6405 	switch (opcode) {
6406 	case I40E_VIRTCHNL_OP_VERSION:
6407 		ixl_vf_version_msg(pf, vf, msg, msg_size);
6408 		break;
6409 	case I40E_VIRTCHNL_OP_RESET_VF:
6410 		ixl_vf_reset_msg(pf, vf, msg, msg_size);
6411 		break;
6412 	case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
6413 		ixl_vf_get_resources_msg(pf, vf, msg, msg_size);
6414 		break;
6415 	case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
6416 		ixl_vf_config_vsi_msg(pf, vf, msg, msg_size);
6417 		break;
6418 	case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
6419 		ixl_vf_config_irq_msg(pf, vf, msg, msg_size);
6420 		break;
6421 	case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
6422 		ixl_vf_enable_queues_msg(pf, vf, msg, msg_size);
6423 		break;
6424 	case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
6425 		ixl_vf_disable_queues_msg(pf, vf, msg, msg_size);
6426 		break;
6427 	case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
6428 		ixl_vf_add_mac_msg(pf, vf, msg, msg_size);
6429 		break;
6430 	case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
6431 		ixl_vf_del_mac_msg(pf, vf, msg, msg_size);
6432 		break;
6433 	case I40E_VIRTCHNL_OP_ADD_VLAN:
6434 		ixl_vf_add_vlan_msg(pf, vf, msg, msg_size);
6435 		break;
6436 	case I40E_VIRTCHNL_OP_DEL_VLAN:
6437 		ixl_vf_del_vlan_msg(pf, vf, msg, msg_size);
6438 		break;
6439 	case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
6440 		ixl_vf_config_promisc_msg(pf, vf, msg, msg_size);
6441 		break;
6442 	case I40E_VIRTCHNL_OP_GET_STATS:
6443 		ixl_vf_get_stats_msg(pf, vf, msg, msg_size);
6444 		break;
6445 
6446 	/* These two opcodes have been superseded by CONFIG_VSI_QUEUES. */
6447 	case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
6448 	case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
6449 	default:
6450 		i40e_send_vf_nack(pf, vf, opcode, I40E_ERR_NOT_IMPLEMENTED);
6451 		break;
6452 	}
6453 }
6454 
6455 /* Handle any VFs that have reset themselves via a Function Level Reset(FLR). */
6456 static void
6457 ixl_handle_vflr(void *arg, int pending)
6458 {
6459 	struct ixl_pf *pf;
6460 	struct i40e_hw *hw;
6461 	uint16_t global_vf_num;
6462 	uint32_t vflrstat_index, vflrstat_mask, vflrstat, icr0;
6463 	int i;
6464 
6465 	pf = arg;
6466 	hw = &pf->hw;
6467 
6468 	IXL_PF_LOCK(pf);
6469 	for (i = 0; i < pf->num_vfs; i++) {
6470 		global_vf_num = hw->func_caps.vf_base_id + i;
6471 
6472 		vflrstat_index = IXL_GLGEN_VFLRSTAT_INDEX(global_vf_num);
6473 		vflrstat_mask = IXL_GLGEN_VFLRSTAT_MASK(global_vf_num);
6474 		vflrstat = rd32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index));
6475 		if (vflrstat & vflrstat_mask) {
6476 			wr32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index),
6477 			    vflrstat_mask);
6478 
6479 			ixl_reinit_vf(pf, &pf->vfs[i]);
6480 		}
6481 	}
6482 
6483 	icr0 = rd32(hw, I40E_PFINT_ICR0_ENA);
6484 	icr0 |= I40E_PFINT_ICR0_ENA_VFLR_MASK;
6485 	wr32(hw, I40E_PFINT_ICR0_ENA, icr0);
6486 	ixl_flush(hw);
6487 
6488 	IXL_PF_UNLOCK(pf);
6489 }
6490 
6491 static int
6492 ixl_adminq_err_to_errno(enum i40e_admin_queue_err err)
6493 {
6494 
6495 	switch (err) {
6496 	case I40E_AQ_RC_EPERM:
6497 		return (EPERM);
6498 	case I40E_AQ_RC_ENOENT:
6499 		return (ENOENT);
6500 	case I40E_AQ_RC_ESRCH:
6501 		return (ESRCH);
6502 	case I40E_AQ_RC_EINTR:
6503 		return (EINTR);
6504 	case I40E_AQ_RC_EIO:
6505 		return (EIO);
6506 	case I40E_AQ_RC_ENXIO:
6507 		return (ENXIO);
6508 	case I40E_AQ_RC_E2BIG:
6509 		return (E2BIG);
6510 	case I40E_AQ_RC_EAGAIN:
6511 		return (EAGAIN);
6512 	case I40E_AQ_RC_ENOMEM:
6513 		return (ENOMEM);
6514 	case I40E_AQ_RC_EACCES:
6515 		return (EACCES);
6516 	case I40E_AQ_RC_EFAULT:
6517 		return (EFAULT);
6518 	case I40E_AQ_RC_EBUSY:
6519 		return (EBUSY);
6520 	case I40E_AQ_RC_EEXIST:
6521 		return (EEXIST);
6522 	case I40E_AQ_RC_EINVAL:
6523 		return (EINVAL);
6524 	case I40E_AQ_RC_ENOTTY:
6525 		return (ENOTTY);
6526 	case I40E_AQ_RC_ENOSPC:
6527 		return (ENOSPC);
6528 	case I40E_AQ_RC_ENOSYS:
6529 		return (ENOSYS);
6530 	case I40E_AQ_RC_ERANGE:
6531 		return (ERANGE);
6532 	case I40E_AQ_RC_EFLUSHED:
6533 		return (EINVAL);	/* No exact equivalent in errno.h */
6534 	case I40E_AQ_RC_BAD_ADDR:
6535 		return (EFAULT);
6536 	case I40E_AQ_RC_EMODE:
6537 		return (EPERM);
6538 	case I40E_AQ_RC_EFBIG:
6539 		return (EFBIG);
6540 	default:
6541 		return (EINVAL);
6542 	}
6543 }
6544 
6545 static int
6546 ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params)
6547 {
6548 	struct ixl_pf *pf;
6549 	struct i40e_hw *hw;
6550 	struct ixl_vsi *pf_vsi;
6551 	enum i40e_status_code ret;
6552 	int i, error;
6553 
6554 	pf = device_get_softc(dev);
6555 	hw = &pf->hw;
6556 	pf_vsi = &pf->vsi;
6557 
6558 	IXL_PF_LOCK(pf);
6559 	pf->vfs = malloc(sizeof(struct ixl_vf) * num_vfs, M_IXL, M_NOWAIT |
6560 	    M_ZERO);
6561 
6562 	if (pf->vfs == NULL) {
6563 		error = ENOMEM;
6564 		goto fail;
6565 	}
6566 
6567 	for (i = 0; i < num_vfs; i++)
6568 		sysctl_ctx_init(&pf->vfs[i].ctx);
6569 
6570 	ret = i40e_aq_add_veb(hw, pf_vsi->uplink_seid, pf_vsi->seid,
6571 	    1, FALSE, FALSE, &pf->veb_seid, NULL);
6572 	if (ret != I40E_SUCCESS) {
6573 		error = ixl_adminq_err_to_errno(hw->aq.asq_last_status);
6574 		device_printf(dev, "add_veb failed; code=%d error=%d", ret,
6575 		    error);
6576 		goto fail;
6577 	}
6578 
6579 	ixl_configure_msix(pf);
6580 	ixl_enable_adminq(hw);
6581 
6582 	pf->num_vfs = num_vfs;
6583 	IXL_PF_UNLOCK(pf);
6584 	return (0);
6585 
6586 fail:
6587 	free(pf->vfs, M_IXL);
6588 	pf->vfs = NULL;
6589 	IXL_PF_UNLOCK(pf);
6590 	return (error);
6591 }
6592 
6593 static void
6594 ixl_iov_uninit(device_t dev)
6595 {
6596 	struct ixl_pf *pf;
6597 	struct i40e_hw *hw;
6598 	struct ixl_vsi *vsi;
6599 	struct ifnet *ifp;
6600 	struct ixl_vf *vfs;
6601 	int i, num_vfs;
6602 
6603 	pf = device_get_softc(dev);
6604 	hw = &pf->hw;
6605 	vsi = &pf->vsi;
6606 	ifp = vsi->ifp;
6607 
6608 	IXL_PF_LOCK(pf);
6609 	for (i = 0; i < pf->num_vfs; i++) {
6610 		if (pf->vfs[i].vsi.seid != 0)
6611 			i40e_aq_delete_element(hw, pf->vfs[i].vsi.seid, NULL);
6612 	}
6613 
6614 	if (pf->veb_seid != 0) {
6615 		i40e_aq_delete_element(hw, pf->veb_seid, NULL);
6616 		pf->veb_seid = 0;
6617 	}
6618 
6619 	if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
6620 		ixl_disable_intr(vsi);
6621 
6622 	vfs = pf->vfs;
6623 	num_vfs = pf->num_vfs;
6624 
6625 	pf->vfs = NULL;
6626 	pf->num_vfs = 0;
6627 	IXL_PF_UNLOCK(pf);
6628 
6629 	/* Do this after the unlock as sysctl_ctx_free might sleep. */
6630 	for (i = 0; i < num_vfs; i++)
6631 		sysctl_ctx_free(&vfs[i].ctx);
6632 	free(vfs, M_IXL);
6633 }
6634 
6635 static int
6636 ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params)
6637 {
6638 	char sysctl_name[QUEUE_NAME_LEN];
6639 	struct ixl_pf *pf;
6640 	struct ixl_vf *vf;
6641 	const void *mac;
6642 	size_t size;
6643 	int error;
6644 
6645 	pf = device_get_softc(dev);
6646 	vf = &pf->vfs[vfnum];
6647 
6648 	IXL_PF_LOCK(pf);
6649 	vf->vf_num = vfnum;
6650 
6651 	vf->vsi.back = pf;
6652 	vf->vf_flags = VF_FLAG_ENABLED;
6653 	SLIST_INIT(&vf->vsi.ftl);
6654 
6655 	error = ixl_vf_setup_vsi(pf, vf);
6656 	if (error != 0)
6657 		goto out;
6658 
6659 	if (nvlist_exists_binary(params, "mac-addr")) {
6660 		mac = nvlist_get_binary(params, "mac-addr", &size);
6661 		bcopy(mac, vf->mac, ETHER_ADDR_LEN);
6662 
6663 		if (nvlist_get_bool(params, "allow-set-mac"))
6664 			vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
6665 	} else
6666 		/*
6667 		 * If the administrator has not specified a MAC address then
6668 		 * we must allow the VF to choose one.
6669 		 */
6670 		vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
6671 
6672 	if (nvlist_get_bool(params, "mac-anti-spoof"))
6673 		vf->vf_flags |= VF_FLAG_MAC_ANTI_SPOOF;
6674 
6675 	if (nvlist_get_bool(params, "allow-promisc"))
6676 		vf->vf_flags |= VF_FLAG_PROMISC_CAP;
6677 
6678 	vf->vf_flags |= VF_FLAG_VLAN_CAP;
6679 
6680 	ixl_reset_vf(pf, vf);
6681 out:
6682 	IXL_PF_UNLOCK(pf);
6683 	if (error == 0) {
6684 		snprintf(sysctl_name, sizeof(sysctl_name), "vf%d", vfnum);
6685 		ixl_add_vsi_sysctls(pf, &vf->vsi, &vf->ctx, sysctl_name);
6686 	}
6687 
6688 	return (error);
6689 }
6690 #endif /* PCI_IOV */
6691