xref: /freebsd/sys/dev/ixl/if_ixl.c (revision f5e9c916afed4a948fe5c03bfaee038d165e12ab)
1 /******************************************************************************
2 
3   Copyright (c) 2013-2015, Intel Corporation
4   All rights reserved.
5 
6   Redistribution and use in source and binary forms, with or without
7   modification, are permitted provided that the following conditions are met:
8 
9    1. Redistributions of source code must retain the above copyright notice,
10       this list of conditions and the following disclaimer.
11 
12    2. Redistributions in binary form must reproduce the above copyright
13       notice, this list of conditions and the following disclaimer in the
14       documentation and/or other materials provided with the distribution.
15 
16    3. Neither the name of the Intel Corporation nor the names of its
17       contributors may be used to endorse or promote products derived from
18       this software without specific prior written permission.
19 
20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30   POSSIBILITY OF SUCH DAMAGE.
31 
32 ******************************************************************************/
33 /*$FreeBSD$*/
34 
35 #ifndef IXL_STANDALONE_BUILD
36 #include "opt_inet.h"
37 #include "opt_inet6.h"
38 #include "opt_rss.h"
39 #endif
40 
41 #include "ixl.h"
42 #include "ixl_pf.h"
43 
44 #ifdef RSS
45 #include <net/rss_config.h>
46 #endif
47 
48 /*********************************************************************
49  *  Driver version
50  *********************************************************************/
51 char ixl_driver_version[] = "1.4.3";
52 
53 /*********************************************************************
54  *  PCI Device ID Table
55  *
56  *  Used by probe to select devices to load on
57  *  Last field stores an index into ixl_strings
58  *  Last entry must be all 0s
59  *
60  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
61  *********************************************************************/
62 
63 static ixl_vendor_info_t ixl_vendor_info_array[] =
64 {
65 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710, 0, 0, 0},
66 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B, 0, 0, 0},
67 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C, 0, 0, 0},
68 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A, 0, 0, 0},
69 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B, 0, 0, 0},
70 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C, 0, 0, 0},
71 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T, 0, 0, 0},
72 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T4, 0, 0, 0},
73 #ifdef X722_SUPPORT
74 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_X722, 0, 0, 0},
75 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_1G_BASE_T_X722, 0, 0, 0},
76 	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T_X722, 0, 0, 0},
77 #endif
78 	/* required last entry */
79 	{0, 0, 0, 0, 0}
80 };
81 
82 /*********************************************************************
83  *  Table of branding strings
84  *********************************************************************/
85 
86 static char    *ixl_strings[] = {
87 	"Intel(R) Ethernet Connection XL710 Driver"
88 };
89 
90 
91 /*********************************************************************
92  *  Function prototypes
93  *********************************************************************/
94 static int      ixl_probe(device_t);
95 static int      ixl_attach(device_t);
96 static int      ixl_detach(device_t);
97 static int      ixl_shutdown(device_t);
98 static int	ixl_get_hw_capabilities(struct ixl_pf *);
99 static void	ixl_cap_txcsum_tso(struct ixl_vsi *, struct ifnet *, int);
100 static int      ixl_ioctl(struct ifnet *, u_long, caddr_t);
101 static void	ixl_init(void *);
102 static void	ixl_init_locked(struct ixl_pf *);
103 static void     ixl_stop(struct ixl_pf *);
104 static void     ixl_media_status(struct ifnet *, struct ifmediareq *);
105 static int      ixl_media_change(struct ifnet *);
106 static void     ixl_update_link_status(struct ixl_pf *);
107 static int      ixl_allocate_pci_resources(struct ixl_pf *);
108 static u16	ixl_get_bus_info(struct i40e_hw *, device_t);
109 static int	ixl_setup_stations(struct ixl_pf *);
110 static int	ixl_switch_config(struct ixl_pf *);
111 static int	ixl_initialize_vsi(struct ixl_vsi *);
112 static int	ixl_assign_vsi_msix(struct ixl_pf *);
113 static int	ixl_assign_vsi_legacy(struct ixl_pf *);
114 static int	ixl_init_msix(struct ixl_pf *);
115 static void	ixl_configure_msix(struct ixl_pf *);
116 static void	ixl_configure_itr(struct ixl_pf *);
117 static void	ixl_configure_legacy(struct ixl_pf *);
118 static void	ixl_init_taskqueues(struct ixl_pf *);
119 static void	ixl_free_taskqueues(struct ixl_pf *);
120 static void	ixl_free_pci_resources(struct ixl_pf *);
121 static void	ixl_local_timer(void *);
122 static int	ixl_setup_interface(device_t, struct ixl_vsi *);
123 static void	ixl_link_event(struct ixl_pf *, struct i40e_arq_event_info *);
124 static void	ixl_config_rss(struct ixl_vsi *);
125 static void	ixl_set_queue_rx_itr(struct ixl_queue *);
126 static void	ixl_set_queue_tx_itr(struct ixl_queue *);
127 static int	ixl_set_advertised_speeds(struct ixl_pf *, int);
128 
129 static int	ixl_enable_rings(struct ixl_vsi *);
130 static int	ixl_disable_rings(struct ixl_vsi *);
131 static void	ixl_enable_intr(struct ixl_vsi *);
132 static void	ixl_disable_intr(struct ixl_vsi *);
133 static void	ixl_disable_rings_intr(struct ixl_vsi *);
134 
135 static void     ixl_enable_adminq(struct i40e_hw *);
136 static void     ixl_disable_adminq(struct i40e_hw *);
137 static void     ixl_enable_queue(struct i40e_hw *, int);
138 static void     ixl_disable_queue(struct i40e_hw *, int);
139 static void     ixl_enable_legacy(struct i40e_hw *);
140 static void     ixl_disable_legacy(struct i40e_hw *);
141 
142 static void     ixl_set_promisc(struct ixl_vsi *);
143 static void     ixl_add_multi(struct ixl_vsi *);
144 static void     ixl_del_multi(struct ixl_vsi *);
145 static void	ixl_register_vlan(void *, struct ifnet *, u16);
146 static void	ixl_unregister_vlan(void *, struct ifnet *, u16);
147 static void	ixl_setup_vlan_filters(struct ixl_vsi *);
148 
149 static void	ixl_init_filters(struct ixl_vsi *);
150 static void	ixl_reconfigure_filters(struct ixl_vsi *vsi);
151 static void	ixl_add_filter(struct ixl_vsi *, u8 *, s16 vlan);
152 static void	ixl_del_filter(struct ixl_vsi *, u8 *, s16 vlan);
153 static void	ixl_add_hw_filters(struct ixl_vsi *, int, int);
154 static void	ixl_del_hw_filters(struct ixl_vsi *, int);
155 static struct ixl_mac_filter *
156 		ixl_find_filter(struct ixl_vsi *, u8 *, s16);
157 static void	ixl_add_mc_filter(struct ixl_vsi *, u8 *);
158 static void	ixl_free_mac_filters(struct ixl_vsi *vsi);
159 
160 
161 /* Sysctl debug interface */
162 #ifdef IXL_DEBUG_SYSCTL
163 static int	ixl_debug_info(SYSCTL_HANDLER_ARGS);
164 static void	ixl_print_debug_info(struct ixl_pf *);
165 #endif
166 
167 /* The MSI/X Interrupt handlers */
168 static void	ixl_intr(void *);
169 static void	ixl_msix_que(void *);
170 static void	ixl_msix_adminq(void *);
171 static void	ixl_handle_mdd_event(struct ixl_pf *);
172 
173 /* Deferred interrupt tasklets */
174 static void	ixl_do_adminq(void *, int);
175 
176 /* Sysctl handlers */
177 static int	ixl_set_flowcntl(SYSCTL_HANDLER_ARGS);
178 static int	ixl_set_advertise(SYSCTL_HANDLER_ARGS);
179 static int	ixl_current_speed(SYSCTL_HANDLER_ARGS);
180 static int	ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS);
181 
182 /* Statistics */
183 static void     ixl_add_hw_stats(struct ixl_pf *);
184 static void	ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *,
185 		    struct sysctl_oid_list *, struct i40e_hw_port_stats *);
186 static void	ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *,
187 		    struct sysctl_oid_list *,
188 		    struct i40e_eth_stats *);
189 static void	ixl_update_stats_counters(struct ixl_pf *);
190 static void	ixl_update_eth_stats(struct ixl_vsi *);
191 static void	ixl_update_vsi_stats(struct ixl_vsi *);
192 static void	ixl_pf_reset_stats(struct ixl_pf *);
193 static void	ixl_vsi_reset_stats(struct ixl_vsi *);
194 static void	ixl_stat_update48(struct i40e_hw *, u32, u32, bool,
195 		    u64 *, u64 *);
196 static void	ixl_stat_update32(struct i40e_hw *, u32, bool,
197 		    u64 *, u64 *);
198 
199 #ifdef IXL_DEBUG_SYSCTL
200 static int 	ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS);
201 static int	ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS);
202 static int	ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS);
203 static int	ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS);
204 static int	ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS);
205 #endif
206 
207 #ifdef PCI_IOV
208 static int	ixl_adminq_err_to_errno(enum i40e_admin_queue_err err);
209 
210 static int	ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t*);
211 static void	ixl_iov_uninit(device_t dev);
212 static int	ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t*);
213 
214 static void	ixl_handle_vf_msg(struct ixl_pf *,
215 		    struct i40e_arq_event_info *);
216 static void	ixl_handle_vflr(void *arg, int pending);
217 
218 static void	ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf);
219 static void	ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf);
220 #endif
221 
222 /*********************************************************************
223  *  FreeBSD Device Interface Entry Points
224  *********************************************************************/
225 
226 static device_method_t ixl_methods[] = {
227 	/* Device interface */
228 	DEVMETHOD(device_probe, ixl_probe),
229 	DEVMETHOD(device_attach, ixl_attach),
230 	DEVMETHOD(device_detach, ixl_detach),
231 	DEVMETHOD(device_shutdown, ixl_shutdown),
232 #ifdef PCI_IOV
233 	DEVMETHOD(pci_iov_init, ixl_iov_init),
234 	DEVMETHOD(pci_iov_uninit, ixl_iov_uninit),
235 	DEVMETHOD(pci_iov_add_vf, ixl_add_vf),
236 #endif
237 	{0, 0}
238 };
239 
240 static driver_t ixl_driver = {
241 	"ixl", ixl_methods, sizeof(struct ixl_pf),
242 };
243 
244 devclass_t ixl_devclass;
245 DRIVER_MODULE(ixl, pci, ixl_driver, ixl_devclass, 0, 0);
246 
247 MODULE_DEPEND(ixl, pci, 1, 1, 1);
248 MODULE_DEPEND(ixl, ether, 1, 1, 1);
249 #ifdef DEV_NETMAP
250 MODULE_DEPEND(ixl, netmap, 1, 1, 1);
251 #endif /* DEV_NETMAP */
252 
253 /*
254 ** Global reset mutex
255 */
256 static struct mtx ixl_reset_mtx;
257 
258 /*
259 ** TUNEABLE PARAMETERS:
260 */
261 
262 static SYSCTL_NODE(_hw, OID_AUTO, ixl, CTLFLAG_RD, 0,
263                    "IXL driver parameters");
264 
265 /*
266  * MSIX should be the default for best performance,
267  * but this allows it to be forced off for testing.
268  */
269 static int ixl_enable_msix = 1;
270 TUNABLE_INT("hw.ixl.enable_msix", &ixl_enable_msix);
271 SYSCTL_INT(_hw_ixl, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixl_enable_msix, 0,
272     "Enable MSI-X interrupts");
273 
274 /*
275 ** Number of descriptors per ring:
276 **   - TX and RX are the same size
277 */
278 static int ixl_ringsz = DEFAULT_RING;
279 TUNABLE_INT("hw.ixl.ringsz", &ixl_ringsz);
280 SYSCTL_INT(_hw_ixl, OID_AUTO, ring_size, CTLFLAG_RDTUN,
281     &ixl_ringsz, 0, "Descriptor Ring Size");
282 
283 /*
284 ** This can be set manually, if left as 0 the
285 ** number of queues will be calculated based
286 ** on cpus and msix vectors available.
287 */
288 int ixl_max_queues = 0;
289 TUNABLE_INT("hw.ixl.max_queues", &ixl_max_queues);
290 SYSCTL_INT(_hw_ixl, OID_AUTO, max_queues, CTLFLAG_RDTUN,
291     &ixl_max_queues, 0, "Number of Queues");
292 
293 /*
294 ** Controls for Interrupt Throttling
295 **	- true/false for dynamic adjustment
296 ** 	- default values for static ITR
297 */
298 int ixl_dynamic_rx_itr = 0;
299 TUNABLE_INT("hw.ixl.dynamic_rx_itr", &ixl_dynamic_rx_itr);
300 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN,
301     &ixl_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate");
302 
303 int ixl_dynamic_tx_itr = 0;
304 TUNABLE_INT("hw.ixl.dynamic_tx_itr", &ixl_dynamic_tx_itr);
305 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN,
306     &ixl_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate");
307 
308 int ixl_rx_itr = IXL_ITR_8K;
309 TUNABLE_INT("hw.ixl.rx_itr", &ixl_rx_itr);
310 SYSCTL_INT(_hw_ixl, OID_AUTO, rx_itr, CTLFLAG_RDTUN,
311     &ixl_rx_itr, 0, "RX Interrupt Rate");
312 
313 int ixl_tx_itr = IXL_ITR_4K;
314 TUNABLE_INT("hw.ixl.tx_itr", &ixl_tx_itr);
315 SYSCTL_INT(_hw_ixl, OID_AUTO, tx_itr, CTLFLAG_RDTUN,
316     &ixl_tx_itr, 0, "TX Interrupt Rate");
317 
318 #ifdef IXL_FDIR
319 static int ixl_enable_fdir = 1;
320 TUNABLE_INT("hw.ixl.enable_fdir", &ixl_enable_fdir);
321 /* Rate at which we sample */
322 int ixl_atr_rate = 20;
323 TUNABLE_INT("hw.ixl.atr_rate", &ixl_atr_rate);
324 #endif
325 
326 #ifdef DEV_NETMAP
327 #define NETMAP_IXL_MAIN /* only bring in one part of the netmap code */
328 #include <dev/netmap/if_ixl_netmap.h>
329 #endif /* DEV_NETMAP */
330 
331 static char *ixl_fc_string[6] = {
332 	"None",
333 	"Rx",
334 	"Tx",
335 	"Full",
336 	"Priority",
337 	"Default"
338 };
339 
340 static MALLOC_DEFINE(M_IXL, "ixl", "ixl driver allocations");
341 
342 static uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] =
343     {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
344 
345 /*********************************************************************
346  *  Device identification routine
347  *
348  *  ixl_probe determines if the driver should be loaded on
349  *  the hardware based on PCI vendor/device id of the device.
350  *
351  *  return BUS_PROBE_DEFAULT on success, positive on failure
352  *********************************************************************/
353 
354 static int
355 ixl_probe(device_t dev)
356 {
357 	ixl_vendor_info_t *ent;
358 
359 	u16	pci_vendor_id, pci_device_id;
360 	u16	pci_subvendor_id, pci_subdevice_id;
361 	char	device_name[256];
362 	static bool lock_init = FALSE;
363 
364 	INIT_DEBUGOUT("ixl_probe: begin");
365 
366 	pci_vendor_id = pci_get_vendor(dev);
367 	if (pci_vendor_id != I40E_INTEL_VENDOR_ID)
368 		return (ENXIO);
369 
370 	pci_device_id = pci_get_device(dev);
371 	pci_subvendor_id = pci_get_subvendor(dev);
372 	pci_subdevice_id = pci_get_subdevice(dev);
373 
374 	ent = ixl_vendor_info_array;
375 	while (ent->vendor_id != 0) {
376 		if ((pci_vendor_id == ent->vendor_id) &&
377 		    (pci_device_id == ent->device_id) &&
378 
379 		    ((pci_subvendor_id == ent->subvendor_id) ||
380 		     (ent->subvendor_id == 0)) &&
381 
382 		    ((pci_subdevice_id == ent->subdevice_id) ||
383 		     (ent->subdevice_id == 0))) {
384 			sprintf(device_name, "%s, Version - %s",
385 				ixl_strings[ent->index],
386 				ixl_driver_version);
387 			device_set_desc_copy(dev, device_name);
388 			/* One shot mutex init */
389 			if (lock_init == FALSE) {
390 				lock_init = TRUE;
391 				mtx_init(&ixl_reset_mtx,
392 				    "ixl_reset",
393 				    "IXL RESET Lock", MTX_DEF);
394 			}
395 			return (BUS_PROBE_DEFAULT);
396 		}
397 		ent++;
398 	}
399 	return (ENXIO);
400 }
401 
402 /*********************************************************************
403  *  Device initialization routine
404  *
405  *  The attach entry point is called when the driver is being loaded.
406  *  This routine identifies the type of hardware, allocates all resources
407  *  and initializes the hardware.
408  *
409  *  return 0 on success, positive on failure
410  *********************************************************************/
411 
412 static int
413 ixl_attach(device_t dev)
414 {
415 	struct ixl_pf	*pf;
416 	struct i40e_hw	*hw;
417 	struct ixl_vsi *vsi;
418 	u16		bus;
419 	int             error = 0;
420 #ifdef PCI_IOV
421 	nvlist_t	*pf_schema, *vf_schema;
422 	int		iov_error;
423 #endif
424 
425 	INIT_DEBUGOUT("ixl_attach: begin");
426 
427 	/* Allocate, clear, and link in our primary soft structure */
428 	pf = device_get_softc(dev);
429 	pf->dev = pf->osdep.dev = dev;
430 	hw = &pf->hw;
431 
432 	/*
433 	** Note this assumes we have a single embedded VSI,
434 	** this could be enhanced later to allocate multiple
435 	*/
436 	vsi = &pf->vsi;
437 	vsi->dev = pf->dev;
438 
439 	/* Core Lock Init*/
440 	IXL_PF_LOCK_INIT(pf, device_get_nameunit(dev));
441 
442 	/* Set up the timer callout */
443 	callout_init_mtx(&pf->timer, &pf->pf_mtx, 0);
444 
445 	/* Set up sysctls */
446 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
447 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
448 	    OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
449 	    pf, 0, ixl_set_flowcntl, "I", "Flow Control");
450 
451 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
452 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
453 	    OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
454 	    pf, 0, ixl_set_advertise, "I", "Advertised Speed");
455 
456 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
457 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
458 	    OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD,
459 	    pf, 0, ixl_current_speed, "A", "Current Port Speed");
460 
461 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
462 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
463 	    OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD,
464 	    pf, 0, ixl_sysctl_show_fw, "A", "Firmware version");
465 
466 	SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
467 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
468 	    OID_AUTO, "rx_itr", CTLFLAG_RW,
469 	    &ixl_rx_itr, IXL_ITR_8K, "RX ITR");
470 
471 	SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
472 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
473 	    OID_AUTO, "dynamic_rx_itr", CTLFLAG_RW,
474 	    &ixl_dynamic_rx_itr, 0, "Dynamic RX ITR");
475 
476 	SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
477 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
478 	    OID_AUTO, "tx_itr", CTLFLAG_RW,
479 	    &ixl_tx_itr, IXL_ITR_4K, "TX ITR");
480 
481 	SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
482 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
483 	    OID_AUTO, "dynamic_tx_itr", CTLFLAG_RW,
484 	    &ixl_dynamic_tx_itr, 0, "Dynamic TX ITR");
485 
486 #ifdef IXL_DEBUG_SYSCTL
487 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
488 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
489 	    OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, pf, 0,
490 	    ixl_debug_info, "I", "Debug Information");
491 
492 	/* Debug shared-code message level */
493 	SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
494 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
495 	    OID_AUTO, "debug_mask", CTLFLAG_RW,
496 	    &pf->hw.debug_mask, 0, "Debug Message Level");
497 
498 	SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
499 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
500 	    OID_AUTO, "vc_debug_level", CTLFLAG_RW, &pf->vc_debug_lvl,
501 	    0, "PF/VF Virtual Channel debug level");
502 
503 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
504 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
505 	    OID_AUTO, "link_status", CTLTYPE_STRING | CTLFLAG_RD,
506 	    pf, 0, ixl_sysctl_link_status, "A", "Current Link Status");
507 
508 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
509 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
510 	    OID_AUTO, "phy_abilities", CTLTYPE_STRING | CTLFLAG_RD,
511 	    pf, 0, ixl_sysctl_phy_abilities, "A", "PHY Abilities");
512 
513 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
514 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
515 	    OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD,
516 	    pf, 0, ixl_sysctl_sw_filter_list, "A", "SW Filter List");
517 
518 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
519 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
520 	    OID_AUTO, "hw_res_alloc", CTLTYPE_STRING | CTLFLAG_RD,
521 	    pf, 0, ixl_sysctl_hw_res_alloc, "A", "HW Resource Allocation");
522 
523 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
524 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
525 	    OID_AUTO, "switch_config", CTLTYPE_STRING | CTLFLAG_RD,
526 	    pf, 0, ixl_sysctl_switch_config, "A", "HW Switch Configuration");
527 #endif
528 
529 	/* Save off the PCI information */
530 	hw->vendor_id = pci_get_vendor(dev);
531 	hw->device_id = pci_get_device(dev);
532 	hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
533 	hw->subsystem_vendor_id =
534 	    pci_read_config(dev, PCIR_SUBVEND_0, 2);
535 	hw->subsystem_device_id =
536 	    pci_read_config(dev, PCIR_SUBDEV_0, 2);
537 
538 	hw->bus.device = pci_get_slot(dev);
539 	hw->bus.func = pci_get_function(dev);
540 
541 	pf->vc_debug_lvl = 1;
542 
543 	/* Do PCI setup - map BAR0, etc */
544 	if (ixl_allocate_pci_resources(pf)) {
545 		device_printf(dev, "Allocation of PCI resources failed\n");
546 		error = ENXIO;
547 		goto err_out;
548 	}
549 
550 	/* Establish a clean starting point */
551 	i40e_clear_hw(hw);
552 	error = i40e_pf_reset(hw);
553 	if (error) {
554 		device_printf(dev,"PF reset failure %x\n", error);
555 		error = EIO;
556 		goto err_out;
557 	}
558 
559 	/* Set admin queue parameters */
560 	hw->aq.num_arq_entries = IXL_AQ_LEN;
561 	hw->aq.num_asq_entries = IXL_AQ_LEN;
562 	hw->aq.arq_buf_size = IXL_AQ_BUFSZ;
563 	hw->aq.asq_buf_size = IXL_AQ_BUFSZ;
564 
565 	/* Initialize the shared code */
566 	error = i40e_init_shared_code(hw);
567 	if (error) {
568 		device_printf(dev,"Unable to initialize the shared code\n");
569 		error = EIO;
570 		goto err_out;
571 	}
572 
573 	/* Set up the admin queue */
574 	error = i40e_init_adminq(hw);
575 	if (error) {
576 		device_printf(dev, "The driver for the device stopped "
577 		    "because the NVM image is newer than expected.\n"
578 		    "You must install the most recent version of "
579 		    " the network driver.\n");
580 		goto err_out;
581 	}
582 	device_printf(dev, "%s\n", ixl_fw_version_str(hw));
583 
584         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
585 	    hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR)
586 		device_printf(dev, "The driver for the device detected "
587 		    "a newer version of the NVM image than expected.\n"
588 		    "Please install the most recent version of the network driver.\n");
589 	else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR ||
590 	    hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1))
591 		device_printf(dev, "The driver for the device detected "
592 		    "an older version of the NVM image than expected.\n"
593 		    "Please update the NVM image.\n");
594 
595 	/* Clear PXE mode */
596 	i40e_clear_pxe_mode(hw);
597 
598 	/* Get capabilities from the device */
599 	error = ixl_get_hw_capabilities(pf);
600 	if (error) {
601 		device_printf(dev, "HW capabilities failure!\n");
602 		goto err_get_cap;
603 	}
604 
605 	/* Set up host memory cache */
606 	error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
607 	    hw->func_caps.num_rx_qp, 0, 0);
608 	if (error) {
609 		device_printf(dev, "init_lan_hmc failed: %d\n", error);
610 		goto err_get_cap;
611 	}
612 
613 	error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
614 	if (error) {
615 		device_printf(dev, "configure_lan_hmc failed: %d\n", error);
616 		goto err_mac_hmc;
617 	}
618 
619 	/* Disable LLDP from the firmware */
620 	i40e_aq_stop_lldp(hw, TRUE, NULL);
621 
622 	i40e_get_mac_addr(hw, hw->mac.addr);
623 	error = i40e_validate_mac_addr(hw->mac.addr);
624 	if (error) {
625 		device_printf(dev, "validate_mac_addr failed: %d\n", error);
626 		goto err_mac_hmc;
627 	}
628 	bcopy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN);
629 	i40e_get_port_mac_addr(hw, hw->mac.port_addr);
630 
631 	/* Set up VSI and queues */
632 	if (ixl_setup_stations(pf) != 0) {
633 		device_printf(dev, "setup stations failed!\n");
634 		error = ENOMEM;
635 		goto err_mac_hmc;
636 	}
637 
638 	/* Initialize mac filter list for VSI */
639 	SLIST_INIT(&vsi->ftl);
640 
641 	/* Set up interrupt routing here */
642 	if (pf->msix > 1)
643 		error = ixl_assign_vsi_msix(pf);
644 	else
645 		error = ixl_assign_vsi_legacy(pf);
646 	if (error)
647 		goto err_mac_hmc;
648 
649 	if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) ||
650 	    (hw->aq.fw_maj_ver < 4)) {
651 		i40e_msec_delay(75);
652 		error = i40e_aq_set_link_restart_an(hw, TRUE, NULL);
653 		if (error)
654 			device_printf(dev, "link restart failed, aq_err=%d\n",
655 			    pf->hw.aq.asq_last_status);
656 	}
657 
658 	/* Determine link state */
659 	i40e_aq_get_link_info(hw, TRUE, NULL, NULL);
660 	i40e_get_link_status(hw, &pf->link_up);
661 
662 	/* Setup OS specific network interface */
663 	if (ixl_setup_interface(dev, vsi) != 0) {
664 		device_printf(dev, "interface setup failed!\n");
665 		error = EIO;
666 		goto err_late;
667 	}
668 
669 	error = ixl_switch_config(pf);
670 	if (error) {
671 		device_printf(dev, "Initial switch config failed: %d\n", error);
672 		goto err_late;
673 	}
674 
675 	/* Limit phy interrupts to link and modules failure */
676 	error = i40e_aq_set_phy_int_mask(hw, ~(I40E_AQ_EVENT_LINK_UPDOWN |
677 		I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
678 	if (error)
679 		device_printf(dev, "set phy mask failed: %d\n", error);
680 
681 	/* Get the bus configuration and set the shared code */
682 	bus = ixl_get_bus_info(hw, dev);
683 	i40e_set_pci_config_data(hw, bus);
684 
685 	/* Initialize taskqueues */
686 	ixl_init_taskqueues(pf);
687 
688 	/* Initialize statistics */
689 	ixl_pf_reset_stats(pf);
690 	ixl_update_stats_counters(pf);
691 	ixl_add_hw_stats(pf);
692 
693 	/* Register for VLAN events */
694 	vsi->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
695 	    ixl_register_vlan, vsi, EVENTHANDLER_PRI_FIRST);
696 	vsi->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
697 	    ixl_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST);
698 
699 #ifdef PCI_IOV
700 	/* SR-IOV is only supported when MSI-X is in use. */
701 	if (pf->msix > 1) {
702 		pf_schema = pci_iov_schema_alloc_node();
703 		vf_schema = pci_iov_schema_alloc_node();
704 		pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL);
705 		pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof",
706 		    IOV_SCHEMA_HASDEFAULT, TRUE);
707 		pci_iov_schema_add_bool(vf_schema, "allow-set-mac",
708 		    IOV_SCHEMA_HASDEFAULT, FALSE);
709 		pci_iov_schema_add_bool(vf_schema, "allow-promisc",
710 		    IOV_SCHEMA_HASDEFAULT, FALSE);
711 
712 		iov_error = pci_iov_attach(dev, pf_schema, vf_schema);
713 		if (iov_error != 0)
714 			device_printf(dev,
715 			    "Failed to initialize SR-IOV (error=%d)\n",
716 			    iov_error);
717 	}
718 #endif
719 
720 #ifdef DEV_NETMAP
721 	ixl_netmap_attach(vsi);
722 #endif /* DEV_NETMAP */
723 	INIT_DEBUGOUT("ixl_attach: end");
724 	return (0);
725 
726 err_late:
727 	if (vsi->ifp != NULL)
728 		if_free(vsi->ifp);
729 err_mac_hmc:
730 	i40e_shutdown_lan_hmc(hw);
731 err_get_cap:
732 	i40e_shutdown_adminq(hw);
733 err_out:
734 	ixl_free_pci_resources(pf);
735 	ixl_free_vsi(vsi);
736 	IXL_PF_LOCK_DESTROY(pf);
737 	return (error);
738 }
739 
740 /*********************************************************************
741  *  Device removal routine
742  *
743  *  The detach entry point is called when the driver is being removed.
744  *  This routine stops the adapter and deallocates all the resources
745  *  that were allocated for driver operation.
746  *
747  *  return 0 on success, positive on failure
748  *********************************************************************/
749 
750 static int
751 ixl_detach(device_t dev)
752 {
753 	struct ixl_pf		*pf = device_get_softc(dev);
754 	struct i40e_hw		*hw = &pf->hw;
755 	struct ixl_vsi		*vsi = &pf->vsi;
756 	i40e_status		status;
757 #ifdef PCI_IOV
758 	int			error;
759 #endif
760 
761 	INIT_DEBUGOUT("ixl_detach: begin");
762 
763 	/* Make sure VLANS are not using driver */
764 	if (vsi->ifp->if_vlantrunk != NULL) {
765 		device_printf(dev,"Vlan in use, detach first\n");
766 		return (EBUSY);
767 	}
768 
769 #ifdef PCI_IOV
770 	error = pci_iov_detach(dev);
771 	if (error != 0) {
772 		device_printf(dev, "SR-IOV in use; detach first.\n");
773 		return (error);
774 	}
775 #endif
776 
777 	ether_ifdetach(vsi->ifp);
778 	if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING) {
779 		IXL_PF_LOCK(pf);
780 		ixl_stop(pf);
781 		IXL_PF_UNLOCK(pf);
782 	}
783 
784 	ixl_free_taskqueues(pf);
785 
786 	/* Shutdown LAN HMC */
787 	status = i40e_shutdown_lan_hmc(hw);
788 	if (status)
789 		device_printf(dev,
790 		    "Shutdown LAN HMC failed with code %d\n", status);
791 
792 	/* Shutdown admin queue */
793 	status = i40e_shutdown_adminq(hw);
794 	if (status)
795 		device_printf(dev,
796 		    "Shutdown Admin queue failed with code %d\n", status);
797 
798 	/* Unregister VLAN events */
799 	if (vsi->vlan_attach != NULL)
800 		EVENTHANDLER_DEREGISTER(vlan_config, vsi->vlan_attach);
801 	if (vsi->vlan_detach != NULL)
802 		EVENTHANDLER_DEREGISTER(vlan_unconfig, vsi->vlan_detach);
803 
804 	callout_drain(&pf->timer);
805 #ifdef DEV_NETMAP
806 	netmap_detach(vsi->ifp);
807 #endif /* DEV_NETMAP */
808 	ixl_free_pci_resources(pf);
809 	bus_generic_detach(dev);
810 	if_free(vsi->ifp);
811 	ixl_free_vsi(vsi);
812 	IXL_PF_LOCK_DESTROY(pf);
813 	return (0);
814 }
815 
816 /*********************************************************************
817  *
818  *  Shutdown entry point
819  *
820  **********************************************************************/
821 
822 static int
823 ixl_shutdown(device_t dev)
824 {
825 	struct ixl_pf *pf = device_get_softc(dev);
826 	IXL_PF_LOCK(pf);
827 	ixl_stop(pf);
828 	IXL_PF_UNLOCK(pf);
829 	return (0);
830 }
831 
832 
833 /*********************************************************************
834  *
835  *  Get the hardware capabilities
836  *
837  **********************************************************************/
838 
839 static int
840 ixl_get_hw_capabilities(struct ixl_pf *pf)
841 {
842 	struct i40e_aqc_list_capabilities_element_resp *buf;
843 	struct i40e_hw	*hw = &pf->hw;
844 	device_t 	dev = pf->dev;
845 	int             error, len;
846 	u16		needed;
847 	bool		again = TRUE;
848 
849 	len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
850 retry:
851 	if (!(buf = (struct i40e_aqc_list_capabilities_element_resp *)
852 	    malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO))) {
853 		device_printf(dev, "Unable to allocate cap memory\n");
854                 return (ENOMEM);
855 	}
856 
857 	/* This populates the hw struct */
858         error = i40e_aq_discover_capabilities(hw, buf, len,
859 	    &needed, i40e_aqc_opc_list_func_capabilities, NULL);
860 	free(buf, M_DEVBUF);
861 	if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) &&
862 	    (again == TRUE)) {
863 		/* retry once with a larger buffer */
864 		again = FALSE;
865 		len = needed;
866 		goto retry;
867 	} else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
868 		device_printf(dev, "capability discovery failed: %d\n",
869 		    pf->hw.aq.asq_last_status);
870 		return (ENODEV);
871 	}
872 
873 	/* Capture this PF's starting queue pair */
874 	pf->qbase = hw->func_caps.base_queue;
875 
876 #ifdef IXL_DEBUG
877 	device_printf(dev,"pf_id=%d, num_vfs=%d, msix_pf=%d, "
878 	    "msix_vf=%d, fd_g=%d, fd_b=%d, tx_qp=%d rx_qp=%d qbase=%d\n",
879 	    hw->pf_id, hw->func_caps.num_vfs,
880 	    hw->func_caps.num_msix_vectors,
881 	    hw->func_caps.num_msix_vectors_vf,
882 	    hw->func_caps.fd_filters_guaranteed,
883 	    hw->func_caps.fd_filters_best_effort,
884 	    hw->func_caps.num_tx_qp,
885 	    hw->func_caps.num_rx_qp,
886 	    hw->func_caps.base_queue);
887 #endif
888 	return (error);
889 }
890 
891 static void
892 ixl_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask)
893 {
894 	device_t 	dev = vsi->dev;
895 
896 	/* Enable/disable TXCSUM/TSO4 */
897 	if (!(ifp->if_capenable & IFCAP_TXCSUM)
898 	    && !(ifp->if_capenable & IFCAP_TSO4)) {
899 		if (mask & IFCAP_TXCSUM) {
900 			ifp->if_capenable |= IFCAP_TXCSUM;
901 			/* enable TXCSUM, restore TSO if previously enabled */
902 			if (vsi->flags & IXL_FLAGS_KEEP_TSO4) {
903 				vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
904 				ifp->if_capenable |= IFCAP_TSO4;
905 			}
906 		}
907 		else if (mask & IFCAP_TSO4) {
908 			ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4);
909 			vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
910 			device_printf(dev,
911 			    "TSO4 requires txcsum, enabling both...\n");
912 		}
913 	} else if((ifp->if_capenable & IFCAP_TXCSUM)
914 	    && !(ifp->if_capenable & IFCAP_TSO4)) {
915 		if (mask & IFCAP_TXCSUM)
916 			ifp->if_capenable &= ~IFCAP_TXCSUM;
917 		else if (mask & IFCAP_TSO4)
918 			ifp->if_capenable |= IFCAP_TSO4;
919 	} else if((ifp->if_capenable & IFCAP_TXCSUM)
920 	    && (ifp->if_capenable & IFCAP_TSO4)) {
921 		if (mask & IFCAP_TXCSUM) {
922 			vsi->flags |= IXL_FLAGS_KEEP_TSO4;
923 			ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4);
924 			device_printf(dev,
925 			    "TSO4 requires txcsum, disabling both...\n");
926 		} else if (mask & IFCAP_TSO4)
927 			ifp->if_capenable &= ~IFCAP_TSO4;
928 	}
929 
930 	/* Enable/disable TXCSUM_IPV6/TSO6 */
931 	if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6)
932 	    && !(ifp->if_capenable & IFCAP_TSO6)) {
933 		if (mask & IFCAP_TXCSUM_IPV6) {
934 			ifp->if_capenable |= IFCAP_TXCSUM_IPV6;
935 			if (vsi->flags & IXL_FLAGS_KEEP_TSO6) {
936 				vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
937 				ifp->if_capenable |= IFCAP_TSO6;
938 			}
939 		} else if (mask & IFCAP_TSO6) {
940 			ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
941 			vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
942 			device_printf(dev,
943 			    "TSO6 requires txcsum6, enabling both...\n");
944 		}
945 	} else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
946 	    && !(ifp->if_capenable & IFCAP_TSO6)) {
947 		if (mask & IFCAP_TXCSUM_IPV6)
948 			ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6;
949 		else if (mask & IFCAP_TSO6)
950 			ifp->if_capenable |= IFCAP_TSO6;
951 	} else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
952 	    && (ifp->if_capenable & IFCAP_TSO6)) {
953 		if (mask & IFCAP_TXCSUM_IPV6) {
954 			vsi->flags |= IXL_FLAGS_KEEP_TSO6;
955 			ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
956 			device_printf(dev,
957 			    "TSO6 requires txcsum6, disabling both...\n");
958 		} else if (mask & IFCAP_TSO6)
959 			ifp->if_capenable &= ~IFCAP_TSO6;
960 	}
961 }
962 
963 /*********************************************************************
964  *  Ioctl entry point
965  *
966  *  ixl_ioctl is called when the user wants to configure the
967  *  interface.
968  *
969  *  return 0 on success, positive on failure
970  **********************************************************************/
971 
972 static int
973 ixl_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
974 {
975 	struct ixl_vsi	*vsi = ifp->if_softc;
976 	struct ixl_pf	*pf = vsi->back;
977 	struct ifreq	*ifr = (struct ifreq *) data;
978 #if defined(INET) || defined(INET6)
979 	struct ifaddr *ifa = (struct ifaddr *)data;
980 	bool		avoid_reset = FALSE;
981 #endif
982 	int             error = 0;
983 
984 	switch (command) {
985 
986         case SIOCSIFADDR:
987 #ifdef INET
988 		if (ifa->ifa_addr->sa_family == AF_INET)
989 			avoid_reset = TRUE;
990 #endif
991 #ifdef INET6
992 		if (ifa->ifa_addr->sa_family == AF_INET6)
993 			avoid_reset = TRUE;
994 #endif
995 #if defined(INET) || defined(INET6)
996 		/*
997 		** Calling init results in link renegotiation,
998 		** so we avoid doing it when possible.
999 		*/
1000 		if (avoid_reset) {
1001 			ifp->if_flags |= IFF_UP;
1002 			if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1003 				ixl_init(pf);
1004 #ifdef INET
1005 			if (!(ifp->if_flags & IFF_NOARP))
1006 				arp_ifinit(ifp, ifa);
1007 #endif
1008 		} else
1009 			error = ether_ioctl(ifp, command, data);
1010 		break;
1011 #endif
1012 	case SIOCSIFMTU:
1013 		IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
1014 		if (ifr->ifr_mtu > IXL_MAX_FRAME -
1015 		   ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) {
1016 			error = EINVAL;
1017 		} else {
1018 			IXL_PF_LOCK(pf);
1019 			ifp->if_mtu = ifr->ifr_mtu;
1020 			vsi->max_frame_size =
1021 				ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
1022 			    + ETHER_VLAN_ENCAP_LEN;
1023 			ixl_init_locked(pf);
1024 			IXL_PF_UNLOCK(pf);
1025 		}
1026 		break;
1027 	case SIOCSIFFLAGS:
1028 		IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
1029 		IXL_PF_LOCK(pf);
1030 		if (ifp->if_flags & IFF_UP) {
1031 			if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1032 				if ((ifp->if_flags ^ pf->if_flags) &
1033 				    (IFF_PROMISC | IFF_ALLMULTI)) {
1034 					ixl_set_promisc(vsi);
1035 				}
1036 			} else
1037 				ixl_init_locked(pf);
1038 		} else
1039 			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1040 				ixl_stop(pf);
1041 		pf->if_flags = ifp->if_flags;
1042 		IXL_PF_UNLOCK(pf);
1043 		break;
1044 	case SIOCADDMULTI:
1045 		IOCTL_DEBUGOUT("ioctl: SIOCADDMULTI");
1046 		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1047 			IXL_PF_LOCK(pf);
1048 			ixl_disable_intr(vsi);
1049 			ixl_add_multi(vsi);
1050 			ixl_enable_intr(vsi);
1051 			IXL_PF_UNLOCK(pf);
1052 		}
1053 		break;
1054 	case SIOCDELMULTI:
1055 		IOCTL_DEBUGOUT("ioctl: SIOCDELMULTI");
1056 		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1057 			IXL_PF_LOCK(pf);
1058 			ixl_disable_intr(vsi);
1059 			ixl_del_multi(vsi);
1060 			ixl_enable_intr(vsi);
1061 			IXL_PF_UNLOCK(pf);
1062 		}
1063 		break;
1064 	case SIOCSIFMEDIA:
1065 	case SIOCGIFMEDIA:
1066 #ifdef IFM_ETH_XTYPE
1067 	case SIOCGIFXMEDIA:
1068 #endif
1069 		IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1070 		error = ifmedia_ioctl(ifp, ifr, &vsi->media, command);
1071 		break;
1072 	case SIOCSIFCAP:
1073 	{
1074 		int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1075 		IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1076 
1077 		ixl_cap_txcsum_tso(vsi, ifp, mask);
1078 
1079 		if (mask & IFCAP_RXCSUM)
1080 			ifp->if_capenable ^= IFCAP_RXCSUM;
1081 		if (mask & IFCAP_RXCSUM_IPV6)
1082 			ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1083 		if (mask & IFCAP_LRO)
1084 			ifp->if_capenable ^= IFCAP_LRO;
1085 		if (mask & IFCAP_VLAN_HWTAGGING)
1086 			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1087 		if (mask & IFCAP_VLAN_HWFILTER)
1088 			ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1089 		if (mask & IFCAP_VLAN_HWTSO)
1090 			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1091 		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1092 			IXL_PF_LOCK(pf);
1093 			ixl_init_locked(pf);
1094 			IXL_PF_UNLOCK(pf);
1095 		}
1096 		VLAN_CAPABILITIES(ifp);
1097 
1098 		break;
1099 	}
1100 
1101 	default:
1102 		IOCTL_DEBUGOUT("ioctl: UNKNOWN (0x%X)\n", (int)command);
1103 		error = ether_ioctl(ifp, command, data);
1104 		break;
1105 	}
1106 
1107 	return (error);
1108 }
1109 
1110 
1111 /*********************************************************************
1112  *  Init entry point
1113  *
1114  *  This routine is used in two ways. It is used by the stack as
1115  *  init entry point in network interface structure. It is also used
1116  *  by the driver as a hw/sw initialization routine to get to a
1117  *  consistent state.
1118  *
1119  *  return 0 on success, positive on failure
1120  **********************************************************************/
1121 
1122 static void
1123 ixl_init_locked(struct ixl_pf *pf)
1124 {
1125 	struct i40e_hw	*hw = &pf->hw;
1126 	struct ixl_vsi	*vsi = &pf->vsi;
1127 	struct ifnet	*ifp = vsi->ifp;
1128 	device_t 	dev = pf->dev;
1129 	struct i40e_filter_control_settings	filter;
1130 	u8		tmpaddr[ETHER_ADDR_LEN];
1131 	int		ret;
1132 
1133 	mtx_assert(&pf->pf_mtx, MA_OWNED);
1134 	INIT_DEBUGOUT("ixl_init: begin");
1135 	ixl_stop(pf);
1136 
1137 	/* Get the latest mac address... User might use a LAA */
1138 	bcopy(IF_LLADDR(vsi->ifp), tmpaddr,
1139 	      I40E_ETH_LENGTH_OF_ADDRESS);
1140 	if (!cmp_etheraddr(hw->mac.addr, tmpaddr) &&
1141 	    (i40e_validate_mac_addr(tmpaddr) == I40E_SUCCESS)) {
1142 		ixl_del_filter(vsi, hw->mac.addr, IXL_VLAN_ANY);
1143 		bcopy(tmpaddr, hw->mac.addr,
1144 		    I40E_ETH_LENGTH_OF_ADDRESS);
1145 		ret = i40e_aq_mac_address_write(hw,
1146 		    I40E_AQC_WRITE_TYPE_LAA_ONLY,
1147 		    hw->mac.addr, NULL);
1148 		if (ret) {
1149 			device_printf(dev, "LLA address"
1150 			 "change failed!!\n");
1151 			return;
1152 		} else {
1153 			ixl_add_filter(vsi, hw->mac.addr, IXL_VLAN_ANY);
1154 		}
1155 	}
1156 
1157 	/* Set the various hardware offload abilities */
1158 	ifp->if_hwassist = 0;
1159 	if (ifp->if_capenable & IFCAP_TSO)
1160 		ifp->if_hwassist |= CSUM_TSO;
1161 	if (ifp->if_capenable & IFCAP_TXCSUM)
1162 		ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1163 	if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
1164 		ifp->if_hwassist |= (CSUM_TCP_IPV6 | CSUM_UDP_IPV6);
1165 
1166 	/* Set up the device filtering */
1167 	bzero(&filter, sizeof(filter));
1168 	filter.enable_ethtype = TRUE;
1169 	filter.enable_macvlan = TRUE;
1170 #ifdef IXL_FDIR
1171 	filter.enable_fdir = TRUE;
1172 #endif
1173 	filter.hash_lut_size = I40E_HASH_LUT_SIZE_512;
1174 	if (i40e_set_filter_control(hw, &filter))
1175 		device_printf(dev, "set_filter_control() failed\n");
1176 
1177 	/* Set up RSS */
1178 	ixl_config_rss(vsi);
1179 
1180 	/*
1181 	** Prepare the VSI: rings, hmc contexts, etc...
1182 	*/
1183 	if (ixl_initialize_vsi(vsi)) {
1184 		device_printf(dev, "initialize vsi failed!!\n");
1185 		return;
1186 	}
1187 
1188 	/* Add protocol filters to list */
1189 	ixl_init_filters(vsi);
1190 
1191 	/* Setup vlan's if needed */
1192 	ixl_setup_vlan_filters(vsi);
1193 
1194 	/* Start the local timer */
1195 	callout_reset(&pf->timer, hz, ixl_local_timer, pf);
1196 
1197 	/* Set up MSI/X routing and the ITR settings */
1198 	if (ixl_enable_msix) {
1199 		ixl_configure_msix(pf);
1200 		ixl_configure_itr(pf);
1201 	} else
1202 		ixl_configure_legacy(pf);
1203 
1204 	ixl_enable_rings(vsi);
1205 
1206 	i40e_aq_set_default_vsi(hw, vsi->seid, NULL);
1207 
1208 	ixl_reconfigure_filters(vsi);
1209 
1210 	/* Set MTU in hardware*/
1211 	int aq_error = i40e_aq_set_mac_config(hw, vsi->max_frame_size,
1212 	    TRUE, 0, NULL);
1213 	if (aq_error)
1214 		device_printf(vsi->dev,
1215 			"aq_set_mac_config in init error, code %d\n",
1216 		    aq_error);
1217 
1218 	/* And now turn on interrupts */
1219 	ixl_enable_intr(vsi);
1220 
1221 	/* Now inform the stack we're ready */
1222 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
1223 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1224 
1225 	return;
1226 }
1227 
1228 static void
1229 ixl_init(void *arg)
1230 {
1231 	struct ixl_pf *pf = arg;
1232 
1233 	IXL_PF_LOCK(pf);
1234 	ixl_init_locked(pf);
1235 	IXL_PF_UNLOCK(pf);
1236 	return;
1237 }
1238 
1239 /*
1240 **
1241 ** MSIX Interrupt Handlers and Tasklets
1242 **
1243 */
1244 static void
1245 ixl_handle_que(void *context, int pending)
1246 {
1247 	struct ixl_queue *que = context;
1248 	struct ixl_vsi *vsi = que->vsi;
1249 	struct i40e_hw  *hw = vsi->hw;
1250 	struct tx_ring  *txr = &que->txr;
1251 	struct ifnet    *ifp = vsi->ifp;
1252 	bool		more;
1253 
1254 	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1255 		more = ixl_rxeof(que, IXL_RX_LIMIT);
1256 		IXL_TX_LOCK(txr);
1257 		ixl_txeof(que);
1258 		if (!drbr_empty(ifp, txr->br))
1259 			ixl_mq_start_locked(ifp, txr);
1260 		IXL_TX_UNLOCK(txr);
1261 		if (more) {
1262 			taskqueue_enqueue(que->tq, &que->task);
1263 			return;
1264 		}
1265 	}
1266 
1267 	/* Reenable this interrupt - hmmm */
1268 	ixl_enable_queue(hw, que->me);
1269 	return;
1270 }
1271 
1272 
1273 /*********************************************************************
1274  *
1275  *  Legacy Interrupt Service routine
1276  *
1277  **********************************************************************/
1278 void
1279 ixl_intr(void *arg)
1280 {
1281 	struct ixl_pf		*pf = arg;
1282 	struct i40e_hw		*hw =  &pf->hw;
1283 	struct ixl_vsi		*vsi = &pf->vsi;
1284 	struct ixl_queue	*que = vsi->queues;
1285 	struct ifnet		*ifp = vsi->ifp;
1286 	struct tx_ring		*txr = &que->txr;
1287         u32			reg, icr0, mask;
1288 	bool			more_tx, more_rx;
1289 
1290 	++que->irqs;
1291 
1292 	/* Protect against spurious interrupts */
1293 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1294 		return;
1295 
1296 	icr0 = rd32(hw, I40E_PFINT_ICR0);
1297 
1298 	reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1299 	reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1300 	wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1301 
1302         mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1303 
1304 #ifdef PCI_IOV
1305 	if (icr0 & I40E_PFINT_ICR0_VFLR_MASK)
1306 		taskqueue_enqueue(pf->tq, &pf->vflr_task);
1307 #endif
1308 
1309 	if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
1310 		taskqueue_enqueue(pf->tq, &pf->adminq);
1311 		return;
1312 	}
1313 
1314 	more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1315 
1316 	IXL_TX_LOCK(txr);
1317 	more_tx = ixl_txeof(que);
1318 	if (!drbr_empty(vsi->ifp, txr->br))
1319 		more_tx = 1;
1320 	IXL_TX_UNLOCK(txr);
1321 
1322 	/* re-enable other interrupt causes */
1323 	wr32(hw, I40E_PFINT_ICR0_ENA, mask);
1324 
1325 	/* And now the queues */
1326 	reg = rd32(hw, I40E_QINT_RQCTL(0));
1327 	reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1328 	wr32(hw, I40E_QINT_RQCTL(0), reg);
1329 
1330 	reg = rd32(hw, I40E_QINT_TQCTL(0));
1331 	reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1332 	reg &= ~I40E_PFINT_ICR0_INTEVENT_MASK;
1333 	wr32(hw, I40E_QINT_TQCTL(0), reg);
1334 
1335 	ixl_enable_legacy(hw);
1336 
1337 	return;
1338 }
1339 
1340 
1341 /*********************************************************************
1342  *
1343  *  MSIX VSI Interrupt Service routine
1344  *
1345  **********************************************************************/
1346 void
1347 ixl_msix_que(void *arg)
1348 {
1349 	struct ixl_queue	*que = arg;
1350 	struct ixl_vsi	*vsi = que->vsi;
1351 	struct i40e_hw	*hw = vsi->hw;
1352 	struct tx_ring	*txr = &que->txr;
1353 	bool		more_tx, more_rx;
1354 
1355 	/* Protect against spurious interrupts */
1356 	if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING))
1357 		return;
1358 
1359 	++que->irqs;
1360 
1361 	more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1362 
1363 	IXL_TX_LOCK(txr);
1364 	more_tx = ixl_txeof(que);
1365 	/*
1366 	** Make certain that if the stack
1367 	** has anything queued the task gets
1368 	** scheduled to handle it.
1369 	*/
1370 	if (!drbr_empty(vsi->ifp, txr->br))
1371 		more_tx = 1;
1372 	IXL_TX_UNLOCK(txr);
1373 
1374 	ixl_set_queue_rx_itr(que);
1375 	ixl_set_queue_tx_itr(que);
1376 
1377 	if (more_tx || more_rx)
1378 		taskqueue_enqueue(que->tq, &que->task);
1379 	else
1380 		ixl_enable_queue(hw, que->me);
1381 
1382 	return;
1383 }
1384 
1385 
1386 /*********************************************************************
1387  *
1388  *  MSIX Admin Queue Interrupt Service routine
1389  *
1390  **********************************************************************/
1391 static void
1392 ixl_msix_adminq(void *arg)
1393 {
1394 	struct ixl_pf	*pf = arg;
1395 	struct i40e_hw	*hw = &pf->hw;
1396 	u32		reg, mask;
1397 
1398 	++pf->admin_irq;
1399 
1400 	reg = rd32(hw, I40E_PFINT_ICR0);
1401 	mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1402 
1403 	/* Check on the cause */
1404 	if (reg & I40E_PFINT_ICR0_ADMINQ_MASK)
1405 		mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
1406 
1407 	if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
1408 		ixl_handle_mdd_event(pf);
1409 		mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
1410 	}
1411 
1412 #ifdef PCI_IOV
1413 	if (reg & I40E_PFINT_ICR0_VFLR_MASK) {
1414 		mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
1415 		taskqueue_enqueue(pf->tq, &pf->vflr_task);
1416 	}
1417 #endif
1418 
1419 	reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1420 	reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1421 	wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1422 
1423 	taskqueue_enqueue(pf->tq, &pf->adminq);
1424 	return;
1425 }
1426 
1427 /*********************************************************************
1428  *
1429  *  Media Ioctl callback
1430  *
1431  *  This routine is called whenever the user queries the status of
1432  *  the interface using ifconfig.
1433  *
1434  **********************************************************************/
1435 static void
1436 ixl_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1437 {
1438 	struct ixl_vsi	*vsi = ifp->if_softc;
1439 	struct ixl_pf	*pf = vsi->back;
1440 	struct i40e_hw  *hw = &pf->hw;
1441 
1442 	INIT_DEBUGOUT("ixl_media_status: begin");
1443 	IXL_PF_LOCK(pf);
1444 
1445 	hw->phy.get_link_info = TRUE;
1446 	i40e_get_link_status(hw, &pf->link_up);
1447 	ixl_update_link_status(pf);
1448 
1449 	ifmr->ifm_status = IFM_AVALID;
1450 	ifmr->ifm_active = IFM_ETHER;
1451 
1452 	if (!pf->link_up) {
1453 		IXL_PF_UNLOCK(pf);
1454 		return;
1455 	}
1456 
1457 	ifmr->ifm_status |= IFM_ACTIVE;
1458 	/* Hardware is always full-duplex */
1459 	ifmr->ifm_active |= IFM_FDX;
1460 
1461 	switch (hw->phy.link_info.phy_type) {
1462 		/* 100 M */
1463 		case I40E_PHY_TYPE_100BASE_TX:
1464 			ifmr->ifm_active |= IFM_100_TX;
1465 			break;
1466 		/* 1 G */
1467 		case I40E_PHY_TYPE_1000BASE_T:
1468 			ifmr->ifm_active |= IFM_1000_T;
1469 			break;
1470 		case I40E_PHY_TYPE_1000BASE_SX:
1471 			ifmr->ifm_active |= IFM_1000_SX;
1472 			break;
1473 		case I40E_PHY_TYPE_1000BASE_LX:
1474 			ifmr->ifm_active |= IFM_1000_LX;
1475 			break;
1476 		/* 10 G */
1477 		case I40E_PHY_TYPE_10GBASE_SFPP_CU:
1478 			ifmr->ifm_active |= IFM_10G_TWINAX;
1479 			break;
1480 		case I40E_PHY_TYPE_10GBASE_SR:
1481 			ifmr->ifm_active |= IFM_10G_SR;
1482 			break;
1483 		case I40E_PHY_TYPE_10GBASE_LR:
1484 			ifmr->ifm_active |= IFM_10G_LR;
1485 			break;
1486 		case I40E_PHY_TYPE_10GBASE_T:
1487 			ifmr->ifm_active |= IFM_10G_T;
1488 			break;
1489 		/* 40 G */
1490 		case I40E_PHY_TYPE_40GBASE_CR4:
1491 		case I40E_PHY_TYPE_40GBASE_CR4_CU:
1492 			ifmr->ifm_active |= IFM_40G_CR4;
1493 			break;
1494 		case I40E_PHY_TYPE_40GBASE_SR4:
1495 			ifmr->ifm_active |= IFM_40G_SR4;
1496 			break;
1497 		case I40E_PHY_TYPE_40GBASE_LR4:
1498 			ifmr->ifm_active |= IFM_40G_LR4;
1499 			break;
1500 #ifndef IFM_ETH_XTYPE
1501 		case I40E_PHY_TYPE_1000BASE_KX:
1502 			ifmr->ifm_active |= IFM_1000_CX;
1503 			break;
1504 		case I40E_PHY_TYPE_10GBASE_CR1_CU:
1505 		case I40E_PHY_TYPE_10GBASE_CR1:
1506 			ifmr->ifm_active |= IFM_10G_TWINAX;
1507 			break;
1508 		case I40E_PHY_TYPE_10GBASE_KX4:
1509 			ifmr->ifm_active |= IFM_10G_CX4;
1510 			break;
1511 		case I40E_PHY_TYPE_10GBASE_KR:
1512 			ifmr->ifm_active |= IFM_10G_SR;
1513 			break;
1514 		case I40E_PHY_TYPE_40GBASE_KR4:
1515 		case I40E_PHY_TYPE_XLPPI:
1516 			ifmr->ifm_active |= IFM_40G_SR4;
1517 			break;
1518 #else
1519 		case I40E_PHY_TYPE_1000BASE_KX:
1520 			ifmr->ifm_active |= IFM_1000_KX;
1521 			break;
1522 		/* ERJ: What's the difference between these? */
1523 		case I40E_PHY_TYPE_10GBASE_CR1_CU:
1524 		case I40E_PHY_TYPE_10GBASE_CR1:
1525 			ifmr->ifm_active |= IFM_10G_CR1;
1526 			break;
1527 		case I40E_PHY_TYPE_10GBASE_KX4:
1528 			ifmr->ifm_active |= IFM_10G_KX4;
1529 			break;
1530 		case I40E_PHY_TYPE_10GBASE_KR:
1531 			ifmr->ifm_active |= IFM_10G_KR;
1532 			break;
1533 		case I40E_PHY_TYPE_20GBASE_KR2:
1534 			ifmr->ifm_active |= IFM_20G_KR2;
1535 			break;
1536 		case I40E_PHY_TYPE_40GBASE_KR4:
1537 			ifmr->ifm_active |= IFM_40G_KR4;
1538 			break;
1539 		case I40E_PHY_TYPE_XLPPI:
1540 			ifmr->ifm_active |= IFM_40G_XLPPI;
1541 			break;
1542 #endif
1543 		default:
1544 			ifmr->ifm_active |= IFM_UNKNOWN;
1545 			break;
1546 	}
1547 	/* Report flow control status as well */
1548 	if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX)
1549 		ifmr->ifm_active |= IFM_ETH_TXPAUSE;
1550 	if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX)
1551 		ifmr->ifm_active |= IFM_ETH_RXPAUSE;
1552 
1553 	IXL_PF_UNLOCK(pf);
1554 
1555 	return;
1556 }
1557 
1558 /*********************************************************************
1559  *
1560  *  Media Ioctl callback
1561  *
1562  *  This routine is called when the user changes speed/duplex using
1563  *  media/mediopt option with ifconfig.
1564  *
1565  **********************************************************************/
1566 static int
1567 ixl_media_change(struct ifnet * ifp)
1568 {
1569 	struct ixl_vsi *vsi = ifp->if_softc;
1570 	struct ifmedia *ifm = &vsi->media;
1571 
1572 	INIT_DEBUGOUT("ixl_media_change: begin");
1573 
1574 	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1575 		return (EINVAL);
1576 
1577 	if_printf(ifp, "Media change is currently not supported.\n");
1578 
1579 	return (ENODEV);
1580 }
1581 
1582 
1583 #ifdef IXL_FDIR
1584 /*
1585 ** ATR: Application Targetted Receive - creates a filter
1586 **	based on TX flow info that will keep the receive
1587 **	portion of the flow on the same queue. Based on the
1588 **	implementation this is only available for TCP connections
1589 */
1590 void
1591 ixl_atr(struct ixl_queue *que, struct tcphdr *th, int etype)
1592 {
1593 	struct ixl_vsi			*vsi = que->vsi;
1594 	struct tx_ring			*txr = &que->txr;
1595 	struct i40e_filter_program_desc	*FDIR;
1596 	u32				ptype, dtype;
1597 	int				idx;
1598 
1599 	/* check if ATR is enabled and sample rate */
1600 	if ((!ixl_enable_fdir) || (!txr->atr_rate))
1601 		return;
1602 	/*
1603 	** We sample all TCP SYN/FIN packets,
1604 	** or at the selected sample rate
1605 	*/
1606 	txr->atr_count++;
1607 	if (((th->th_flags & (TH_FIN | TH_SYN)) == 0) &&
1608 	    (txr->atr_count < txr->atr_rate))
1609                 return;
1610 	txr->atr_count = 0;
1611 
1612 	/* Get a descriptor to use */
1613 	idx = txr->next_avail;
1614 	FDIR = (struct i40e_filter_program_desc *) &txr->base[idx];
1615 	if (++idx == que->num_desc)
1616 		idx = 0;
1617 	txr->avail--;
1618 	txr->next_avail = idx;
1619 
1620 	ptype = (que->me << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1621 	    I40E_TXD_FLTR_QW0_QINDEX_MASK;
1622 
1623 	ptype |= (etype == ETHERTYPE_IP) ?
1624 	    (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
1625 	    I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
1626 	    (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
1627 	    I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
1628 
1629 	ptype |= vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
1630 
1631 	dtype = I40E_TX_DESC_DTYPE_FILTER_PROG;
1632 
1633 	/*
1634 	** We use the TCP TH_FIN as a trigger to remove
1635 	** the filter, otherwise its an update.
1636 	*/
1637 	dtype |= (th->th_flags & TH_FIN) ?
1638 	    (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1639 	    I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
1640 	    (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1641 	    I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1642 
1643 	dtype |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
1644 	    I40E_TXD_FLTR_QW1_DEST_SHIFT;
1645 
1646 	dtype |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
1647 	    I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
1648 
1649 	FDIR->qindex_flex_ptype_vsi = htole32(ptype);
1650 	FDIR->dtype_cmd_cntindex = htole32(dtype);
1651 	return;
1652 }
1653 #endif
1654 
1655 
1656 static void
1657 ixl_set_promisc(struct ixl_vsi *vsi)
1658 {
1659 	struct ifnet	*ifp = vsi->ifp;
1660 	struct i40e_hw	*hw = vsi->hw;
1661 	int		err, mcnt = 0;
1662 	bool		uni = FALSE, multi = FALSE;
1663 
1664 	if (ifp->if_flags & IFF_ALLMULTI)
1665                 multi = TRUE;
1666 	else { /* Need to count the multicast addresses */
1667 		struct  ifmultiaddr *ifma;
1668 		if_maddr_rlock(ifp);
1669 		TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1670                         if (ifma->ifma_addr->sa_family != AF_LINK)
1671                                 continue;
1672                         if (mcnt == MAX_MULTICAST_ADDR)
1673                                 break;
1674                         mcnt++;
1675 		}
1676 		if_maddr_runlock(ifp);
1677 	}
1678 
1679 	if (mcnt >= MAX_MULTICAST_ADDR)
1680                 multi = TRUE;
1681         if (ifp->if_flags & IFF_PROMISC)
1682 		uni = TRUE;
1683 
1684 	err = i40e_aq_set_vsi_unicast_promiscuous(hw,
1685 	    vsi->seid, uni, NULL);
1686 	err = i40e_aq_set_vsi_multicast_promiscuous(hw,
1687 	    vsi->seid, multi, NULL);
1688 	return;
1689 }
1690 
1691 /*********************************************************************
1692  * 	Filter Routines
1693  *
1694  *	Routines for multicast and vlan filter management.
1695  *
1696  *********************************************************************/
1697 static void
1698 ixl_add_multi(struct ixl_vsi *vsi)
1699 {
1700 	struct	ifmultiaddr	*ifma;
1701 	struct ifnet		*ifp = vsi->ifp;
1702 	struct i40e_hw		*hw = vsi->hw;
1703 	int			mcnt = 0, flags;
1704 
1705 	IOCTL_DEBUGOUT("ixl_add_multi: begin");
1706 
1707 	if_maddr_rlock(ifp);
1708 	/*
1709 	** First just get a count, to decide if we
1710 	** we simply use multicast promiscuous.
1711 	*/
1712 	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1713 		if (ifma->ifma_addr->sa_family != AF_LINK)
1714 			continue;
1715 		mcnt++;
1716 	}
1717 	if_maddr_runlock(ifp);
1718 
1719 	if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) {
1720 		/* delete existing MC filters */
1721 		ixl_del_hw_filters(vsi, mcnt);
1722 		i40e_aq_set_vsi_multicast_promiscuous(hw,
1723 		    vsi->seid, TRUE, NULL);
1724 		return;
1725 	}
1726 
1727 	mcnt = 0;
1728 	if_maddr_rlock(ifp);
1729 	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1730 		if (ifma->ifma_addr->sa_family != AF_LINK)
1731 			continue;
1732 		ixl_add_mc_filter(vsi,
1733 		    (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr));
1734 		mcnt++;
1735 	}
1736 	if_maddr_runlock(ifp);
1737 	if (mcnt > 0) {
1738 		flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC);
1739 		ixl_add_hw_filters(vsi, flags, mcnt);
1740 	}
1741 
1742 	IOCTL_DEBUGOUT("ixl_add_multi: end");
1743 	return;
1744 }
1745 
1746 static void
1747 ixl_del_multi(struct ixl_vsi *vsi)
1748 {
1749 	struct ifnet		*ifp = vsi->ifp;
1750 	struct ifmultiaddr	*ifma;
1751 	struct ixl_mac_filter	*f;
1752 	int			mcnt = 0;
1753 	bool		match = FALSE;
1754 
1755 	IOCTL_DEBUGOUT("ixl_del_multi: begin");
1756 
1757 	/* Search for removed multicast addresses */
1758 	if_maddr_rlock(ifp);
1759 	SLIST_FOREACH(f, &vsi->ftl, next) {
1760 		if ((f->flags & IXL_FILTER_USED) && (f->flags & IXL_FILTER_MC)) {
1761 			match = FALSE;
1762 			TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1763 				if (ifma->ifma_addr->sa_family != AF_LINK)
1764 					continue;
1765 				u8 *mc_addr = (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
1766 				if (cmp_etheraddr(f->macaddr, mc_addr)) {
1767 					match = TRUE;
1768 					break;
1769 				}
1770 			}
1771 			if (match == FALSE) {
1772 				f->flags |= IXL_FILTER_DEL;
1773 				mcnt++;
1774 			}
1775 		}
1776 	}
1777 	if_maddr_runlock(ifp);
1778 
1779 	if (mcnt > 0)
1780 		ixl_del_hw_filters(vsi, mcnt);
1781 }
1782 
1783 
1784 /*********************************************************************
1785  *  Timer routine
1786  *
1787  *  This routine checks for link status,updates statistics,
1788  *  and runs the watchdog check.
1789  *
1790  **********************************************************************/
1791 
1792 static void
1793 ixl_local_timer(void *arg)
1794 {
1795 	struct ixl_pf		*pf = arg;
1796 	struct i40e_hw		*hw = &pf->hw;
1797 	struct ixl_vsi		*vsi = &pf->vsi;
1798 	struct ixl_queue	*que = vsi->queues;
1799 	device_t		dev = pf->dev;
1800 	int			hung = 0;
1801 	u32			mask;
1802 
1803 	mtx_assert(&pf->pf_mtx, MA_OWNED);
1804 
1805 	/* Fire off the adminq task */
1806 	taskqueue_enqueue(pf->tq, &pf->adminq);
1807 
1808 	/* Update stats */
1809 	ixl_update_stats_counters(pf);
1810 
1811 	/*
1812 	** Check status of the queues
1813 	*/
1814 	mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK |
1815 		I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK);
1816 
1817 	for (int i = 0; i < vsi->num_queues; i++,que++) {
1818 		/* Any queues with outstanding work get a sw irq */
1819 		if (que->busy)
1820 			wr32(hw, I40E_PFINT_DYN_CTLN(que->me), mask);
1821 		/*
1822 		** Each time txeof runs without cleaning, but there
1823 		** are uncleaned descriptors it increments busy. If
1824 		** we get to 5 we declare it hung.
1825 		*/
1826 		if (que->busy == IXL_QUEUE_HUNG) {
1827 			++hung;
1828 			/* Mark the queue as inactive */
1829 			vsi->active_queues &= ~((u64)1 << que->me);
1830 			continue;
1831 		} else {
1832 			/* Check if we've come back from hung */
1833 			if ((vsi->active_queues & ((u64)1 << que->me)) == 0)
1834 				vsi->active_queues |= ((u64)1 << que->me);
1835 		}
1836 		if (que->busy >= IXL_MAX_TX_BUSY) {
1837 #ifdef IXL_DEBUG
1838 			device_printf(dev,"Warning queue %d "
1839 			    "appears to be hung!\n", i);
1840 #endif
1841 			que->busy = IXL_QUEUE_HUNG;
1842 			++hung;
1843 		}
1844 	}
1845 	/* Only reinit if all queues show hung */
1846 	if (hung == vsi->num_queues)
1847 		goto hung;
1848 
1849 	callout_reset(&pf->timer, hz, ixl_local_timer, pf);
1850 	return;
1851 
1852 hung:
1853 	device_printf(dev, "Local Timer: HANG DETECT - Resetting!!\n");
1854 	ixl_init_locked(pf);
1855 }
1856 
1857 /*
1858 ** Note: this routine updates the OS on the link state
1859 **	the real check of the hardware only happens with
1860 **	a link interrupt.
1861 */
1862 static void
1863 ixl_update_link_status(struct ixl_pf *pf)
1864 {
1865 	struct ixl_vsi		*vsi = &pf->vsi;
1866 	struct i40e_hw		*hw = &pf->hw;
1867 	struct ifnet		*ifp = vsi->ifp;
1868 	device_t		dev = pf->dev;
1869 
1870 	if (pf->link_up){
1871 		if (vsi->link_active == FALSE) {
1872 			pf->fc = hw->fc.current_mode;
1873 			if (bootverbose) {
1874 				device_printf(dev,"Link is up %d Gbps %s,"
1875 				    " Flow Control: %s\n",
1876 				    ((pf->link_speed ==
1877 				    I40E_LINK_SPEED_40GB)? 40:10),
1878 				    "Full Duplex", ixl_fc_string[pf->fc]);
1879 			}
1880 			vsi->link_active = TRUE;
1881 			/*
1882 			** Warn user if link speed on NPAR enabled
1883 			** partition is not at least 10GB
1884 			*/
1885 			if (hw->func_caps.npar_enable &&
1886 			   (hw->phy.link_info.link_speed ==
1887 			   I40E_LINK_SPEED_1GB ||
1888 			   hw->phy.link_info.link_speed ==
1889 			   I40E_LINK_SPEED_100MB))
1890 				device_printf(dev, "The partition detected"
1891 				    "link speed that is less than 10Gbps\n");
1892 			if_link_state_change(ifp, LINK_STATE_UP);
1893 		}
1894 	} else { /* Link down */
1895 		if (vsi->link_active == TRUE) {
1896 			if (bootverbose)
1897 				device_printf(dev,"Link is Down\n");
1898 			if_link_state_change(ifp, LINK_STATE_DOWN);
1899 			vsi->link_active = FALSE;
1900 		}
1901 	}
1902 
1903 	return;
1904 }
1905 
1906 /*********************************************************************
1907  *
1908  *  This routine disables all traffic on the adapter by issuing a
1909  *  global reset on the MAC and deallocates TX/RX buffers.
1910  *
1911  **********************************************************************/
1912 
1913 static void
1914 ixl_stop(struct ixl_pf *pf)
1915 {
1916 	struct ixl_vsi	*vsi = &pf->vsi;
1917 	struct ifnet	*ifp = vsi->ifp;
1918 
1919 	mtx_assert(&pf->pf_mtx, MA_OWNED);
1920 
1921 	INIT_DEBUGOUT("ixl_stop: begin\n");
1922 	if (pf->num_vfs == 0)
1923 		ixl_disable_intr(vsi);
1924 	else
1925 		ixl_disable_rings_intr(vsi);
1926 	ixl_disable_rings(vsi);
1927 
1928 	/* Tell the stack that the interface is no longer active */
1929 	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1930 
1931 	/* Stop the local timer */
1932 	callout_stop(&pf->timer);
1933 
1934 	return;
1935 }
1936 
1937 
1938 /*********************************************************************
1939  *
1940  *  Setup MSIX Interrupt resources and handlers for the VSI
1941  *
1942  **********************************************************************/
1943 static int
1944 ixl_assign_vsi_legacy(struct ixl_pf *pf)
1945 {
1946 	device_t        dev = pf->dev;
1947 	struct 		ixl_vsi *vsi = &pf->vsi;
1948 	struct		ixl_queue *que = vsi->queues;
1949 	int 		error, rid = 0;
1950 
1951 	if (pf->msix == 1)
1952 		rid = 1;
1953 	pf->res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
1954 	    &rid, RF_SHAREABLE | RF_ACTIVE);
1955 	if (pf->res == NULL) {
1956 		device_printf(dev,"Unable to allocate"
1957 		    " bus resource: vsi legacy/msi interrupt\n");
1958 		return (ENXIO);
1959 	}
1960 
1961 	/* Set the handler function */
1962 	error = bus_setup_intr(dev, pf->res,
1963 	    INTR_TYPE_NET | INTR_MPSAFE, NULL,
1964 	    ixl_intr, pf, &pf->tag);
1965 	if (error) {
1966 		pf->res = NULL;
1967 		device_printf(dev, "Failed to register legacy/msi handler");
1968 		return (error);
1969 	}
1970 	bus_describe_intr(dev, pf->res, pf->tag, "irq0");
1971 	TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
1972 	TASK_INIT(&que->task, 0, ixl_handle_que, que);
1973 	que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
1974 	    taskqueue_thread_enqueue, &que->tq);
1975 	taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
1976 	    device_get_nameunit(dev));
1977 	TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
1978 
1979 #ifdef PCI_IOV
1980 	TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf);
1981 #endif
1982 
1983 	pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT,
1984 	    taskqueue_thread_enqueue, &pf->tq);
1985 	taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq",
1986 	    device_get_nameunit(dev));
1987 
1988 	return (0);
1989 }
1990 
1991 static void
1992 ixl_init_taskqueues(struct ixl_pf *pf)
1993 {
1994 	struct ixl_vsi *vsi = &pf->vsi;
1995 	struct ixl_queue *que = vsi->queues;
1996 	device_t dev = pf->dev;
1997 #ifdef	RSS
1998 	int cpu_id;
1999 	cpuset_t cpu_mask;
2000 #endif
2001 
2002 	/* Tasklet for Admin Queue */
2003 	TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
2004 #ifdef PCI_IOV
2005 	/* VFLR Tasklet */
2006 	TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf);
2007 #endif
2008 
2009 	/* Create and start PF taskqueue */
2010 	pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT,
2011 	    taskqueue_thread_enqueue, &pf->tq);
2012 	taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq",
2013 	    device_get_nameunit(dev));
2014 
2015 	/* Create queue tasks and start queue taskqueues */
2016 	for (int i = 0; i < vsi->num_queues; i++, que++) {
2017 		TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
2018 		TASK_INIT(&que->task, 0, ixl_handle_que, que);
2019 		que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
2020 		    taskqueue_thread_enqueue, &que->tq);
2021 #ifdef RSS
2022 		cpu_id = rss_getcpu(i % rss_getnumbuckets());
2023 		CPU_SETOF(cpu_id, &cpu_mask);
2024 		taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET,
2025 		    &cpu_mask, "%s (bucket %d)",
2026 		    device_get_nameunit(dev), cpu_id);
2027 #else
2028 		taskqueue_start_threads(&que->tq, 1, PI_NET,
2029 		    "%s (que %d)", device_get_nameunit(dev), que->me);
2030 #endif
2031 	}
2032 }
2033 
2034 static void
2035 ixl_free_taskqueues(struct ixl_pf *pf)
2036 {
2037 	struct ixl_vsi *vsi = &pf->vsi;
2038 	struct ixl_queue *que = vsi->queues;
2039 
2040 	if (pf->tq)
2041 		taskqueue_free(pf->tq);
2042 	for (int i = 0; i < vsi->num_queues; i++, que++) {
2043 		if (que->tq)
2044 			taskqueue_free(que->tq);
2045 	}
2046 }
2047 
2048 /*********************************************************************
2049  *
2050  *  Setup MSIX Interrupt resources and handlers for the VSI
2051  *
2052  **********************************************************************/
2053 static int
2054 ixl_assign_vsi_msix(struct ixl_pf *pf)
2055 {
2056 	device_t	dev = pf->dev;
2057 	struct 		ixl_vsi *vsi = &pf->vsi;
2058 	struct 		ixl_queue *que = vsi->queues;
2059 	struct		tx_ring	 *txr;
2060 	int 		error, rid, vector = 0;
2061 
2062 	/* Admin Que is vector 0*/
2063 	rid = vector + 1;
2064 	pf->res = bus_alloc_resource_any(dev,
2065     	    SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2066 	if (!pf->res) {
2067 		device_printf(dev,"Unable to allocate"
2068     	    " bus resource: Adminq interrupt [%d]\n", rid);
2069 		return (ENXIO);
2070 	}
2071 	/* Set the adminq vector and handler */
2072 	error = bus_setup_intr(dev, pf->res,
2073 	    INTR_TYPE_NET | INTR_MPSAFE, NULL,
2074 	    ixl_msix_adminq, pf, &pf->tag);
2075 	if (error) {
2076 		pf->res = NULL;
2077 		device_printf(dev, "Failed to register Admin que handler");
2078 		return (error);
2079 	}
2080 	bus_describe_intr(dev, pf->res, pf->tag, "aq");
2081 	pf->admvec = vector;
2082 	++vector;
2083 
2084 	/* Now set up the stations */
2085 	for (int i = 0; i < vsi->num_queues; i++, vector++, que++) {
2086 		int cpu_id = i;
2087 		rid = vector + 1;
2088 		txr = &que->txr;
2089 		que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2090 		    RF_SHAREABLE | RF_ACTIVE);
2091 		if (que->res == NULL) {
2092 			device_printf(dev,"Unable to allocate"
2093 		    	    " bus resource: que interrupt [%d]\n", vector);
2094 			return (ENXIO);
2095 		}
2096 		/* Set the handler function */
2097 		error = bus_setup_intr(dev, que->res,
2098 		    INTR_TYPE_NET | INTR_MPSAFE, NULL,
2099 		    ixl_msix_que, que, &que->tag);
2100 		if (error) {
2101 			que->res = NULL;
2102 			device_printf(dev, "Failed to register que handler");
2103 			return (error);
2104 		}
2105 		bus_describe_intr(dev, que->res, que->tag, "q%d", i);
2106 		/* Bind the vector to a CPU */
2107 #ifdef RSS
2108 		cpu_id = rss_getcpu(i % rss_getnumbuckets());
2109 #endif
2110 		bus_bind_intr(dev, que->res, cpu_id);
2111 		que->msix = vector;
2112 	}
2113 
2114 	return (0);
2115 }
2116 
2117 
2118 /*
2119  * Allocate MSI/X vectors
2120  */
2121 static int
2122 ixl_init_msix(struct ixl_pf *pf)
2123 {
2124 	device_t dev = pf->dev;
2125 	int rid, want, vectors, queues, available;
2126 
2127 	/* Override by tuneable */
2128 	if (ixl_enable_msix == 0)
2129 		goto msi;
2130 
2131 	/*
2132 	** When used in a virtualized environment
2133 	** PCI BUSMASTER capability may not be set
2134 	** so explicity set it here and rewrite
2135 	** the ENABLE in the MSIX control register
2136 	** at this point to cause the host to
2137 	** successfully initialize us.
2138 	*/
2139 	{
2140 		u16 pci_cmd_word;
2141 		int msix_ctrl;
2142 		pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2143 		pci_cmd_word |= PCIM_CMD_BUSMASTEREN;
2144 		pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2);
2145 		pci_find_cap(dev, PCIY_MSIX, &rid);
2146 		rid += PCIR_MSIX_CTRL;
2147 		msix_ctrl = pci_read_config(dev, rid, 2);
2148 		msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
2149 		pci_write_config(dev, rid, msix_ctrl, 2);
2150 	}
2151 
2152 	/* First try MSI/X */
2153 	rid = PCIR_BAR(IXL_BAR);
2154 	pf->msix_mem = bus_alloc_resource_any(dev,
2155 	    SYS_RES_MEMORY, &rid, RF_ACTIVE);
2156        	if (!pf->msix_mem) {
2157 		/* May not be enabled */
2158 		device_printf(pf->dev,
2159 		    "Unable to map MSIX table \n");
2160 		goto msi;
2161 	}
2162 
2163 	available = pci_msix_count(dev);
2164 	if (available == 0) { /* system has msix disabled */
2165 		bus_release_resource(dev, SYS_RES_MEMORY,
2166 		    rid, pf->msix_mem);
2167 		pf->msix_mem = NULL;
2168 		goto msi;
2169 	}
2170 
2171 	/* Figure out a reasonable auto config value */
2172 	queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus;
2173 
2174 	/* Override with hardcoded value if it's less than autoconfig count */
2175 	if ((ixl_max_queues != 0) && (ixl_max_queues <= queues))
2176 		queues = ixl_max_queues;
2177 	else if ((ixl_max_queues != 0) && (ixl_max_queues > queues))
2178 		device_printf(dev, "ixl_max_queues > # of cpus, using "
2179 		    "autoconfig amount...\n");
2180 	/* Or limit maximum auto-configured queues to 8 */
2181 	else if ((ixl_max_queues == 0) && (queues > 8))
2182 		queues = 8;
2183 
2184 #ifdef  RSS
2185 	/* If we're doing RSS, clamp at the number of RSS buckets */
2186 	if (queues > rss_getnumbuckets())
2187 		queues = rss_getnumbuckets();
2188 #endif
2189 
2190 	/*
2191 	** Want one vector (RX/TX pair) per queue
2192 	** plus an additional for the admin queue.
2193 	*/
2194 	want = queues + 1;
2195 	if (want <= available)	/* Have enough */
2196 		vectors = want;
2197 	else {
2198                	device_printf(pf->dev,
2199 		    "MSIX Configuration Problem, "
2200 		    "%d vectors available but %d wanted!\n",
2201 		    available, want);
2202 		return (0); /* Will go to Legacy setup */
2203 	}
2204 
2205 	if (pci_alloc_msix(dev, &vectors) == 0) {
2206                	device_printf(pf->dev,
2207 		    "Using MSIX interrupts with %d vectors\n", vectors);
2208 		pf->msix = vectors;
2209 		pf->vsi.num_queues = queues;
2210 #ifdef RSS
2211 		/*
2212 		 * If we're doing RSS, the number of queues needs to
2213 		 * match the number of RSS buckets that are configured.
2214 		 *
2215 		 * + If there's more queues than RSS buckets, we'll end
2216 		 *   up with queues that get no traffic.
2217 		 *
2218 		 * + If there's more RSS buckets than queues, we'll end
2219 		 *   up having multiple RSS buckets map to the same queue,
2220 		 *   so there'll be some contention.
2221 		 */
2222 		if (queues != rss_getnumbuckets()) {
2223 			device_printf(dev,
2224 			    "%s: queues (%d) != RSS buckets (%d)"
2225 			    "; performance will be impacted.\n",
2226 			    __func__, queues, rss_getnumbuckets());
2227 		}
2228 #endif
2229 		return (vectors);
2230 	}
2231 msi:
2232        	vectors = pci_msi_count(dev);
2233 	pf->vsi.num_queues = 1;
2234 	pf->msix = 1;
2235 	ixl_max_queues = 1;
2236 	ixl_enable_msix = 0;
2237        	if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0)
2238                	device_printf(pf->dev,"Using an MSI interrupt\n");
2239 	else {
2240 		pf->msix = 0;
2241                	device_printf(pf->dev,"Using a Legacy interrupt\n");
2242 	}
2243 	return (vectors);
2244 }
2245 
2246 
2247 /*
2248  * Plumb MSI/X vectors
2249  */
2250 static void
2251 ixl_configure_msix(struct ixl_pf *pf)
2252 {
2253 	struct i40e_hw	*hw = &pf->hw;
2254 	struct ixl_vsi *vsi = &pf->vsi;
2255 	u32		reg;
2256 	u16		vector = 1;
2257 
2258 	/* First set up the adminq - vector 0 */
2259 	wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
2260 	rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
2261 
2262 	reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
2263 	    I40E_PFINT_ICR0_ENA_GRST_MASK |
2264 	    I40E_PFINT_ICR0_HMC_ERR_MASK |
2265 	    I40E_PFINT_ICR0_ENA_ADMINQ_MASK |
2266 	    I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
2267 	    I40E_PFINT_ICR0_ENA_VFLR_MASK |
2268 	    I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK;
2269 	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2270 
2271 	wr32(hw, I40E_PFINT_LNKLST0, 0x7FF);
2272 	wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR), 0x003E);
2273 
2274 	wr32(hw, I40E_PFINT_DYN_CTL0,
2275 	    I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
2276 	    I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
2277 
2278 	wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2279 
2280 	/* Next configure the queues */
2281 	for (int i = 0; i < vsi->num_queues; i++, vector++) {
2282 		wr32(hw, I40E_PFINT_DYN_CTLN(i), 0);
2283 		/* First queue type is RX / type 0 */
2284 		wr32(hw, I40E_PFINT_LNKLSTN(i), i);
2285 
2286 		reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
2287 		(IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
2288 		(vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
2289 		(i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
2290 		(I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
2291 		wr32(hw, I40E_QINT_RQCTL(i), reg);
2292 
2293 		reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
2294 		(IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
2295 		(vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
2296 		(IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
2297 		(I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2298 		wr32(hw, I40E_QINT_TQCTL(i), reg);
2299 	}
2300 }
2301 
2302 /*
2303  * Configure for MSI single vector operation
2304  */
2305 static void
2306 ixl_configure_legacy(struct ixl_pf *pf)
2307 {
2308 	struct i40e_hw	*hw = &pf->hw;
2309 	u32		reg;
2310 
2311 
2312 	wr32(hw, I40E_PFINT_ITR0(0), 0);
2313 	wr32(hw, I40E_PFINT_ITR0(1), 0);
2314 
2315 
2316 	/* Setup "other" causes */
2317 	reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK
2318 	    | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK
2319 	    | I40E_PFINT_ICR0_ENA_GRST_MASK
2320 	    | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK
2321 	    | I40E_PFINT_ICR0_ENA_GPIO_MASK
2322 	    | I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK
2323 	    | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK
2324 	    | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK
2325 	    | I40E_PFINT_ICR0_ENA_VFLR_MASK
2326 	    | I40E_PFINT_ICR0_ENA_ADMINQ_MASK
2327 	    ;
2328 	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2329 
2330 	/* SW_ITR_IDX = 0, but don't change INTENA */
2331 	wr32(hw, I40E_PFINT_DYN_CTL0,
2332 	    I40E_PFINT_DYN_CTLN_SW_ITR_INDX_MASK |
2333 	    I40E_PFINT_DYN_CTLN_INTENA_MSK_MASK);
2334 	/* SW_ITR_IDX = 0, OTHER_ITR_IDX = 0 */
2335 	wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2336 
2337 	/* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
2338 	wr32(hw, I40E_PFINT_LNKLST0, 0);
2339 
2340 	/* Associate the queue pair to the vector and enable the q int */
2341 	reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK
2342 	    | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)
2343 	    | (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2344 	wr32(hw, I40E_QINT_RQCTL(0), reg);
2345 
2346 	reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK
2347 	    | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)
2348 	    | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2349 	wr32(hw, I40E_QINT_TQCTL(0), reg);
2350 
2351 	/* Next enable the queue pair */
2352 	reg = rd32(hw, I40E_QTX_ENA(0));
2353 	reg |= I40E_QTX_ENA_QENA_REQ_MASK;
2354 	wr32(hw, I40E_QTX_ENA(0), reg);
2355 
2356 	reg = rd32(hw, I40E_QRX_ENA(0));
2357 	reg |= I40E_QRX_ENA_QENA_REQ_MASK;
2358 	wr32(hw, I40E_QRX_ENA(0), reg);
2359 }
2360 
2361 
2362 /*
2363  * Set the Initial ITR state
2364  */
2365 static void
2366 ixl_configure_itr(struct ixl_pf *pf)
2367 {
2368 	struct i40e_hw		*hw = &pf->hw;
2369 	struct ixl_vsi		*vsi = &pf->vsi;
2370 	struct ixl_queue	*que = vsi->queues;
2371 
2372 	vsi->rx_itr_setting = ixl_rx_itr;
2373 	if (ixl_dynamic_rx_itr)
2374 		vsi->rx_itr_setting |= IXL_ITR_DYNAMIC;
2375 	vsi->tx_itr_setting = ixl_tx_itr;
2376 	if (ixl_dynamic_tx_itr)
2377 		vsi->tx_itr_setting |= IXL_ITR_DYNAMIC;
2378 
2379 	for (int i = 0; i < vsi->num_queues; i++, que++) {
2380 		struct tx_ring	*txr = &que->txr;
2381 		struct rx_ring 	*rxr = &que->rxr;
2382 
2383 		wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i),
2384 		    vsi->rx_itr_setting);
2385 		rxr->itr = vsi->rx_itr_setting;
2386 		rxr->latency = IXL_AVE_LATENCY;
2387 		wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i),
2388 		    vsi->tx_itr_setting);
2389 		txr->itr = vsi->tx_itr_setting;
2390 		txr->latency = IXL_AVE_LATENCY;
2391 	}
2392 }
2393 
2394 
2395 static int
2396 ixl_allocate_pci_resources(struct ixl_pf *pf)
2397 {
2398 	int             rid;
2399 	device_t        dev = pf->dev;
2400 
2401 	rid = PCIR_BAR(0);
2402 	pf->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2403 	    &rid, RF_ACTIVE);
2404 
2405 	if (!(pf->pci_mem)) {
2406 		device_printf(dev,"Unable to allocate bus resource: memory\n");
2407 		return (ENXIO);
2408 	}
2409 
2410 	pf->osdep.mem_bus_space_tag =
2411 		rman_get_bustag(pf->pci_mem);
2412 	pf->osdep.mem_bus_space_handle =
2413 		rman_get_bushandle(pf->pci_mem);
2414 	pf->osdep.mem_bus_space_size = rman_get_size(pf->pci_mem);
2415 	pf->osdep.flush_reg = I40E_GLGEN_STAT;
2416 	pf->hw.hw_addr = (u8 *) &pf->osdep.mem_bus_space_handle;
2417 
2418 	pf->hw.back = &pf->osdep;
2419 
2420 	/*
2421 	** Now setup MSI or MSI/X, should
2422 	** return us the number of supported
2423 	** vectors. (Will be 1 for MSI)
2424 	*/
2425 	pf->msix = ixl_init_msix(pf);
2426 	return (0);
2427 }
2428 
2429 static void
2430 ixl_free_pci_resources(struct ixl_pf * pf)
2431 {
2432 	struct ixl_vsi		*vsi = &pf->vsi;
2433 	struct ixl_queue	*que = vsi->queues;
2434 	device_t		dev = pf->dev;
2435 	int			rid, memrid;
2436 
2437 	memrid = PCIR_BAR(IXL_BAR);
2438 
2439 	/* We may get here before stations are setup */
2440 	if ((!ixl_enable_msix) || (que == NULL))
2441 		goto early;
2442 
2443 	/*
2444 	**  Release all msix VSI resources:
2445 	*/
2446 	for (int i = 0; i < vsi->num_queues; i++, que++) {
2447 		rid = que->msix + 1;
2448 		if (que->tag != NULL) {
2449 			bus_teardown_intr(dev, que->res, que->tag);
2450 			que->tag = NULL;
2451 		}
2452 		if (que->res != NULL)
2453 			bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2454 	}
2455 
2456 early:
2457 	/* Clean the AdminQ interrupt last */
2458 	if (pf->admvec) /* we are doing MSIX */
2459 		rid = pf->admvec + 1;
2460 	else
2461 		(pf->msix != 0) ? (rid = 1):(rid = 0);
2462 
2463 	if (pf->tag != NULL) {
2464 		bus_teardown_intr(dev, pf->res, pf->tag);
2465 		pf->tag = NULL;
2466 	}
2467 	if (pf->res != NULL)
2468 		bus_release_resource(dev, SYS_RES_IRQ, rid, pf->res);
2469 
2470 	if (pf->msix)
2471 		pci_release_msi(dev);
2472 
2473 	if (pf->msix_mem != NULL)
2474 		bus_release_resource(dev, SYS_RES_MEMORY,
2475 		    memrid, pf->msix_mem);
2476 
2477 	if (pf->pci_mem != NULL)
2478 		bus_release_resource(dev, SYS_RES_MEMORY,
2479 		    PCIR_BAR(0), pf->pci_mem);
2480 
2481 	return;
2482 }
2483 
2484 static void
2485 ixl_add_ifmedia(struct ixl_vsi *vsi, u32 phy_type)
2486 {
2487 	/* Display supported media types */
2488 	if (phy_type & (1 << I40E_PHY_TYPE_100BASE_TX))
2489 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL);
2490 
2491 	if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_T))
2492 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL);
2493 	if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_SX))
2494 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
2495 	if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_LX))
2496 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL);
2497 
2498 	if (phy_type & (1 << I40E_PHY_TYPE_XAUI) ||
2499 	    phy_type & (1 << I40E_PHY_TYPE_XFI) ||
2500 	    phy_type & (1 << I40E_PHY_TYPE_10GBASE_SFPP_CU))
2501 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2502 
2503 	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_SR))
2504 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2505 	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_LR))
2506 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL);
2507 	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_T))
2508 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL);
2509 
2510 	if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4) ||
2511 	    phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4_CU) ||
2512 	    phy_type & (1 << I40E_PHY_TYPE_40GBASE_AOC) ||
2513 	    phy_type & (1 << I40E_PHY_TYPE_XLAUI) ||
2514 	    phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2515 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL);
2516 	if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_SR4))
2517 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
2518 	if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_LR4))
2519 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL);
2520 
2521 #ifndef IFM_ETH_XTYPE
2522 	if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX))
2523 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_CX, 0, NULL);
2524 
2525 	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) ||
2526 	    phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1) ||
2527 	    phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC) ||
2528 	    phy_type & (1 << I40E_PHY_TYPE_SFI))
2529 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2530 	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4))
2531 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CX4, 0, NULL);
2532 	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR))
2533 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2534 
2535 	if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2536 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
2537 	if (phy_type & (1 << I40E_PHY_TYPE_XLPPI))
2538 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL);
2539 #else
2540 	if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX))
2541 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_KX, 0, NULL);
2542 
2543 	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU)
2544 	    || phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1))
2545 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CR1, 0, NULL);
2546 	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC))
2547 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX_LONG, 0, NULL);
2548 	if (phy_type & (1 << I40E_PHY_TYPE_SFI))
2549 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SFI, 0, NULL);
2550 	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4))
2551 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KX4, 0, NULL);
2552 	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR))
2553 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KR, 0, NULL);
2554 
2555 	if (phy_type & (1 << I40E_PHY_TYPE_20GBASE_KR2))
2556 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_20G_KR2, 0, NULL);
2557 
2558 	if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2559 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_KR4, 0, NULL);
2560 	if (phy_type & (1 << I40E_PHY_TYPE_XLPPI))
2561 		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_XLPPI, 0, NULL);
2562 #endif
2563 }
2564 
2565 /*********************************************************************
2566  *
2567  *  Setup networking device structure and register an interface.
2568  *
2569  **********************************************************************/
2570 static int
2571 ixl_setup_interface(device_t dev, struct ixl_vsi *vsi)
2572 {
2573 	struct ifnet		*ifp;
2574 	struct i40e_hw		*hw = vsi->hw;
2575 	struct ixl_queue	*que = vsi->queues;
2576 	struct i40e_aq_get_phy_abilities_resp abilities;
2577 	enum i40e_status_code aq_error = 0;
2578 
2579 	INIT_DEBUGOUT("ixl_setup_interface: begin");
2580 
2581 	ifp = vsi->ifp = if_alloc(IFT_ETHER);
2582 	if (ifp == NULL) {
2583 		device_printf(dev, "can not allocate ifnet structure\n");
2584 		return (-1);
2585 	}
2586 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2587 	ifp->if_mtu = ETHERMTU;
2588 	ifp->if_baudrate = IF_Gbps(40);
2589 	ifp->if_init = ixl_init;
2590 	ifp->if_softc = vsi;
2591 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2592 	ifp->if_ioctl = ixl_ioctl;
2593 
2594 #if __FreeBSD_version >= 1100036
2595 	if_setgetcounterfn(ifp, ixl_get_counter);
2596 #endif
2597 
2598 	ifp->if_transmit = ixl_mq_start;
2599 
2600 	ifp->if_qflush = ixl_qflush;
2601 
2602 	ifp->if_snd.ifq_maxlen = que->num_desc - 2;
2603 
2604 	vsi->max_frame_size =
2605 	    ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
2606 	    + ETHER_VLAN_ENCAP_LEN;
2607 
2608 	/*
2609 	 * Tell the upper layer(s) we support long frames.
2610 	 */
2611 	ifp->if_hdrlen = sizeof(struct ether_vlan_header);
2612 
2613 	ifp->if_capabilities |= IFCAP_HWCSUM;
2614 	ifp->if_capabilities |= IFCAP_HWCSUM_IPV6;
2615 	ifp->if_capabilities |= IFCAP_TSO;
2616 	ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2617 	ifp->if_capabilities |= IFCAP_LRO;
2618 
2619 	/* VLAN capabilties */
2620 	ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2621 			     |  IFCAP_VLAN_HWTSO
2622 			     |  IFCAP_VLAN_MTU
2623 			     |  IFCAP_VLAN_HWCSUM;
2624 	ifp->if_capenable = ifp->if_capabilities;
2625 
2626 	/*
2627 	** Don't turn this on by default, if vlans are
2628 	** created on another pseudo device (eg. lagg)
2629 	** then vlan events are not passed thru, breaking
2630 	** operation, but with HW FILTER off it works. If
2631 	** using vlans directly on the ixl driver you can
2632 	** enable this and get full hardware tag filtering.
2633 	*/
2634 	ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2635 
2636 	/*
2637 	 * Specify the media types supported by this adapter and register
2638 	 * callbacks to update media and link information
2639 	 */
2640 	ifmedia_init(&vsi->media, IFM_IMASK, ixl_media_change,
2641 		     ixl_media_status);
2642 
2643 	aq_error = i40e_aq_get_phy_capabilities(hw,
2644 	    FALSE, TRUE, &abilities, NULL);
2645 	/* May need delay to detect fiber correctly */
2646 	if (aq_error == I40E_ERR_UNKNOWN_PHY) {
2647 		i40e_msec_delay(200);
2648 		aq_error = i40e_aq_get_phy_capabilities(hw, FALSE,
2649 		    TRUE, &abilities, NULL);
2650 	}
2651 	if (aq_error) {
2652 		if (aq_error == I40E_ERR_UNKNOWN_PHY)
2653 			device_printf(dev, "Unknown PHY type detected!\n");
2654 		else
2655 			device_printf(dev,
2656 			    "Error getting supported media types, err %d,"
2657 			    " AQ error %d\n", aq_error, hw->aq.asq_last_status);
2658 		return (0);
2659 	}
2660 
2661 	ixl_add_ifmedia(vsi, abilities.phy_type);
2662 
2663 	/* Use autoselect media by default */
2664 	ifmedia_add(&vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2665 	ifmedia_set(&vsi->media, IFM_ETHER | IFM_AUTO);
2666 
2667 	ether_ifattach(ifp, hw->mac.addr);
2668 
2669 	return (0);
2670 }
2671 
2672 /*
2673 ** Run when the Admin Queue gets a
2674 ** link transition interrupt.
2675 */
2676 static void
2677 ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e)
2678 {
2679 	struct i40e_hw	*hw = &pf->hw;
2680 	struct i40e_aqc_get_link_status *status =
2681 	    (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
2682 	bool check;
2683 
2684 	hw->phy.get_link_info = TRUE;
2685 	i40e_get_link_status(hw, &check);
2686 	pf->link_up = check;
2687 #ifdef IXL_DEBUG
2688 	printf("Link is %s\n", check ? "up":"down");
2689 #endif
2690 	/* Report if Unqualified modules are found */
2691 	if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
2692 	    (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
2693 	    (!(status->link_info & I40E_AQ_LINK_UP)))
2694 		device_printf(pf->dev, "Link failed because "
2695 		    "an unqualified module was detected\n");
2696 
2697 	return;
2698 }
2699 
2700 /*********************************************************************
2701  *
2702  *  Get Firmware Switch configuration
2703  *	- this will need to be more robust when more complex
2704  *	  switch configurations are enabled.
2705  *
2706  **********************************************************************/
2707 static int
2708 ixl_switch_config(struct ixl_pf *pf)
2709 {
2710 	struct i40e_hw	*hw = &pf->hw;
2711 	struct ixl_vsi	*vsi = &pf->vsi;
2712 	device_t 	dev = vsi->dev;
2713 	struct i40e_aqc_get_switch_config_resp *sw_config;
2714 	u8	aq_buf[I40E_AQ_LARGE_BUF];
2715 	int	ret;
2716 	u16	next = 0;
2717 
2718 	memset(&aq_buf, 0, sizeof(aq_buf));
2719 	sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
2720 	ret = i40e_aq_get_switch_config(hw, sw_config,
2721 	    sizeof(aq_buf), &next, NULL);
2722 	if (ret) {
2723 		device_printf(dev,"aq_get_switch_config failed (ret=%d)!!\n",
2724 		    ret);
2725 		return (ret);
2726 	}
2727 #ifdef IXL_DEBUG
2728 	device_printf(dev,
2729 	    "Switch config: header reported: %d in structure, %d total\n",
2730     	    sw_config->header.num_reported, sw_config->header.num_total);
2731 	for (int i = 0; i < sw_config->header.num_reported; i++) {
2732 		device_printf(dev,
2733 		    "%d: type=%d seid=%d uplink=%d downlink=%d\n", i,
2734 		    sw_config->element[i].element_type,
2735 		    sw_config->element[i].seid,
2736 		    sw_config->element[i].uplink_seid,
2737 		    sw_config->element[i].downlink_seid);
2738 	}
2739 #endif
2740 	/* Simplified due to a single VSI at the moment */
2741 	vsi->uplink_seid = sw_config->element[0].uplink_seid;
2742 	vsi->downlink_seid = sw_config->element[0].downlink_seid;
2743 	vsi->seid = sw_config->element[0].seid;
2744 	return (ret);
2745 }
2746 
2747 /*********************************************************************
2748  *
2749  *  Initialize the VSI:  this handles contexts, which means things
2750  *  			 like the number of descriptors, buffer size,
2751  *			 plus we init the rings thru this function.
2752  *
2753  **********************************************************************/
2754 static int
2755 ixl_initialize_vsi(struct ixl_vsi *vsi)
2756 {
2757 	struct ixl_pf		*pf = vsi->back;
2758 	struct ixl_queue	*que = vsi->queues;
2759 	device_t		dev = vsi->dev;
2760 	struct i40e_hw		*hw = vsi->hw;
2761 	struct i40e_vsi_context	ctxt;
2762 	int			err = 0;
2763 
2764 	memset(&ctxt, 0, sizeof(ctxt));
2765 	ctxt.seid = vsi->seid;
2766 	if (pf->veb_seid != 0)
2767 		ctxt.uplink_seid = pf->veb_seid;
2768 	ctxt.pf_num = hw->pf_id;
2769 	err = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
2770 	if (err) {
2771 		device_printf(dev,"get vsi params failed %x!!\n", err);
2772 		return (err);
2773 	}
2774 #ifdef IXL_DEBUG
2775 	printf("get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, "
2776 	    "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, "
2777 	    "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid,
2778 	    ctxt.uplink_seid, ctxt.vsi_number,
2779 	    ctxt.vsis_allocated, ctxt.vsis_unallocated,
2780 	    ctxt.flags, ctxt.pf_num, ctxt.vf_num,
2781 	    ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits);
2782 #endif
2783 	/*
2784 	** Set the queue and traffic class bits
2785 	**  - when multiple traffic classes are supported
2786 	**    this will need to be more robust.
2787 	*/
2788 	ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
2789 	ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG;
2790 	/* In contig mode, que_mapping[0] is first queue index used by this VSI */
2791 	ctxt.info.queue_mapping[0] = 0;
2792 	/*
2793 	 * This VSI will only use traffic class 0; start traffic class 0's
2794 	 * queue allocation at queue 0, and assign it 64 (2^6) queues (though
2795 	 * the driver may not use all of them).
2796 	 */
2797 	ctxt.info.tc_mapping[0] = ((0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT)
2798 	    & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) |
2799 	    ((6 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT)
2800 	    & I40E_AQ_VSI_TC_QUE_NUMBER_MASK);
2801 
2802 	/* Set VLAN receive stripping mode */
2803 	ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID;
2804 	ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL;
2805 	if (vsi->ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
2806 	    ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2807 	else
2808 	    ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2809 
2810 	/* Keep copy of VSI info in VSI for statistic counters */
2811 	memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
2812 
2813 	/* Reset VSI statistics */
2814 	ixl_vsi_reset_stats(vsi);
2815 	vsi->hw_filters_add = 0;
2816 	vsi->hw_filters_del = 0;
2817 
2818 	ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF);
2819 
2820 	err = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
2821 	if (err) {
2822 		device_printf(dev,"update vsi params failed %x!!\n",
2823 		   hw->aq.asq_last_status);
2824 		return (err);
2825 	}
2826 
2827 	for (int i = 0; i < vsi->num_queues; i++, que++) {
2828 		struct tx_ring		*txr = &que->txr;
2829 		struct rx_ring 		*rxr = &que->rxr;
2830 		struct i40e_hmc_obj_txq tctx;
2831 		struct i40e_hmc_obj_rxq rctx;
2832 		u32			txctl;
2833 		u16			size;
2834 
2835 
2836 		/* Setup the HMC TX Context  */
2837 		size = que->num_desc * sizeof(struct i40e_tx_desc);
2838 		memset(&tctx, 0, sizeof(struct i40e_hmc_obj_txq));
2839 		tctx.new_context = 1;
2840 		tctx.base = (txr->dma.pa/IXL_TX_CTX_BASE_UNITS);
2841 		tctx.qlen = que->num_desc;
2842 		tctx.fc_ena = 0;
2843 		tctx.rdylist = vsi->info.qs_handle[0]; /* index is TC */
2844 		/* Enable HEAD writeback */
2845 		tctx.head_wb_ena = 1;
2846 		tctx.head_wb_addr = txr->dma.pa +
2847 		    (que->num_desc * sizeof(struct i40e_tx_desc));
2848 		tctx.rdylist_act = 0;
2849 		err = i40e_clear_lan_tx_queue_context(hw, i);
2850 		if (err) {
2851 			device_printf(dev, "Unable to clear TX context\n");
2852 			break;
2853 		}
2854 		err = i40e_set_lan_tx_queue_context(hw, i, &tctx);
2855 		if (err) {
2856 			device_printf(dev, "Unable to set TX context\n");
2857 			break;
2858 		}
2859 		/* Associate the ring with this PF */
2860 		txctl = I40E_QTX_CTL_PF_QUEUE;
2861 		txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
2862 		    I40E_QTX_CTL_PF_INDX_MASK);
2863 		wr32(hw, I40E_QTX_CTL(i), txctl);
2864 		ixl_flush(hw);
2865 
2866 		/* Do ring (re)init */
2867 		ixl_init_tx_ring(que);
2868 
2869 		/* Next setup the HMC RX Context  */
2870 		if (vsi->max_frame_size <= MCLBYTES)
2871 			rxr->mbuf_sz = MCLBYTES;
2872 		else
2873 			rxr->mbuf_sz = MJUMPAGESIZE;
2874 
2875 		u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len;
2876 
2877 		/* Set up an RX context for the HMC */
2878 		memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq));
2879 		rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT;
2880 		/* ignore header split for now */
2881 		rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT;
2882 		rctx.rxmax = (vsi->max_frame_size < max_rxmax) ?
2883 		    vsi->max_frame_size : max_rxmax;
2884 		rctx.dtype = 0;
2885 		rctx.dsize = 1;	/* do 32byte descriptors */
2886 		rctx.hsplit_0 = 0;  /* no HDR split initially */
2887 		rctx.base = (rxr->dma.pa/IXL_RX_CTX_BASE_UNITS);
2888 		rctx.qlen = que->num_desc;
2889 		rctx.tphrdesc_ena = 1;
2890 		rctx.tphwdesc_ena = 1;
2891 		rctx.tphdata_ena = 0;
2892 		rctx.tphhead_ena = 0;
2893 		rctx.lrxqthresh = 2;
2894 		rctx.crcstrip = 1;
2895 		rctx.l2tsel = 1;
2896 		rctx.showiv = 1;
2897 		rctx.fc_ena = 0;
2898 		rctx.prefena = 1;
2899 
2900 		err = i40e_clear_lan_rx_queue_context(hw, i);
2901 		if (err) {
2902 			device_printf(dev,
2903 			    "Unable to clear RX context %d\n", i);
2904 			break;
2905 		}
2906 		err = i40e_set_lan_rx_queue_context(hw, i, &rctx);
2907 		if (err) {
2908 			device_printf(dev, "Unable to set RX context %d\n", i);
2909 			break;
2910 		}
2911 		err = ixl_init_rx_ring(que);
2912 		if (err) {
2913 			device_printf(dev, "Fail in init_rx_ring %d\n", i);
2914 			break;
2915 		}
2916 #ifdef DEV_NETMAP
2917 		/* preserve queue */
2918 		if (vsi->ifp->if_capenable & IFCAP_NETMAP) {
2919 			struct netmap_adapter *na = NA(vsi->ifp);
2920 			struct netmap_kring *kring = &na->rx_rings[i];
2921 			int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
2922 			wr32(vsi->hw, I40E_QRX_TAIL(que->me), t);
2923 		} else
2924 #endif /* DEV_NETMAP */
2925 		wr32(vsi->hw, I40E_QRX_TAIL(que->me), que->num_desc - 1);
2926 	}
2927 	return (err);
2928 }
2929 
2930 
2931 /*********************************************************************
2932  *
2933  *  Free all VSI structs.
2934  *
2935  **********************************************************************/
2936 void
2937 ixl_free_vsi(struct ixl_vsi *vsi)
2938 {
2939 	struct ixl_pf		*pf = (struct ixl_pf *)vsi->back;
2940 	struct ixl_queue	*que = vsi->queues;
2941 
2942 	/* Free station queues */
2943 	for (int i = 0; i < vsi->num_queues; i++, que++) {
2944 		struct tx_ring *txr = &que->txr;
2945 		struct rx_ring *rxr = &que->rxr;
2946 
2947 		if (!mtx_initialized(&txr->mtx)) /* uninitialized */
2948 			continue;
2949 		IXL_TX_LOCK(txr);
2950 		ixl_free_que_tx(que);
2951 		if (txr->base)
2952 			i40e_free_dma_mem(&pf->hw, &txr->dma);
2953 		IXL_TX_UNLOCK(txr);
2954 		IXL_TX_LOCK_DESTROY(txr);
2955 
2956 		if (!mtx_initialized(&rxr->mtx)) /* uninitialized */
2957 			continue;
2958 		IXL_RX_LOCK(rxr);
2959 		ixl_free_que_rx(que);
2960 		if (rxr->base)
2961 			i40e_free_dma_mem(&pf->hw, &rxr->dma);
2962 		IXL_RX_UNLOCK(rxr);
2963 		IXL_RX_LOCK_DESTROY(rxr);
2964 
2965 	}
2966 	free(vsi->queues, M_DEVBUF);
2967 
2968 	/* Free VSI filter list */
2969 	ixl_free_mac_filters(vsi);
2970 }
2971 
2972 static void
2973 ixl_free_mac_filters(struct ixl_vsi *vsi)
2974 {
2975 	struct ixl_mac_filter *f;
2976 
2977 	while (!SLIST_EMPTY(&vsi->ftl)) {
2978 		f = SLIST_FIRST(&vsi->ftl);
2979 		SLIST_REMOVE_HEAD(&vsi->ftl, next);
2980 		free(f, M_DEVBUF);
2981 	}
2982 }
2983 
2984 
2985 /*********************************************************************
2986  *
2987  *  Allocate memory for the VSI (virtual station interface) and their
2988  *  associated queues, rings and the descriptors associated with each,
2989  *  called only once at attach.
2990  *
2991  **********************************************************************/
2992 static int
2993 ixl_setup_stations(struct ixl_pf *pf)
2994 {
2995 	device_t		dev = pf->dev;
2996 	struct ixl_vsi		*vsi;
2997 	struct ixl_queue	*que;
2998 	struct tx_ring		*txr;
2999 	struct rx_ring		*rxr;
3000 	int 			rsize, tsize;
3001 	int			error = I40E_SUCCESS;
3002 
3003 	vsi = &pf->vsi;
3004 	vsi->back = (void *)pf;
3005 	vsi->hw = &pf->hw;
3006 	vsi->id = 0;
3007 	vsi->num_vlans = 0;
3008 	vsi->back = pf;
3009 
3010 	/* Get memory for the station queues */
3011         if (!(vsi->queues =
3012             (struct ixl_queue *) malloc(sizeof(struct ixl_queue) *
3013             vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3014                 device_printf(dev, "Unable to allocate queue memory\n");
3015                 error = ENOMEM;
3016                 goto early;
3017         }
3018 
3019 	for (int i = 0; i < vsi->num_queues; i++) {
3020 		que = &vsi->queues[i];
3021 		que->num_desc = ixl_ringsz;
3022 		que->me = i;
3023 		que->vsi = vsi;
3024 		/* mark the queue as active */
3025 		vsi->active_queues |= (u64)1 << que->me;
3026 		txr = &que->txr;
3027 		txr->que = que;
3028 		txr->tail = I40E_QTX_TAIL(que->me);
3029 
3030 		/* Initialize the TX lock */
3031 		snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
3032 		    device_get_nameunit(dev), que->me);
3033 		mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF);
3034 		/* Create the TX descriptor ring */
3035 		tsize = roundup2((que->num_desc *
3036 		    sizeof(struct i40e_tx_desc)) +
3037 		    sizeof(u32), DBA_ALIGN);
3038 		if (i40e_allocate_dma_mem(&pf->hw,
3039 		    &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) {
3040 			device_printf(dev,
3041 			    "Unable to allocate TX Descriptor memory\n");
3042 			error = ENOMEM;
3043 			goto fail;
3044 		}
3045 		txr->base = (struct i40e_tx_desc *)txr->dma.va;
3046 		bzero((void *)txr->base, tsize);
3047        		/* Now allocate transmit soft structs for the ring */
3048        		if (ixl_allocate_tx_data(que)) {
3049 			device_printf(dev,
3050 			    "Critical Failure setting up TX structures\n");
3051 			error = ENOMEM;
3052 			goto fail;
3053        		}
3054 		/* Allocate a buf ring */
3055 		txr->br = buf_ring_alloc(4096, M_DEVBUF,
3056 		    M_WAITOK, &txr->mtx);
3057 		if (txr->br == NULL) {
3058 			device_printf(dev,
3059 			    "Critical Failure setting up TX buf ring\n");
3060 			error = ENOMEM;
3061 			goto fail;
3062        		}
3063 
3064 		/*
3065 		 * Next the RX queues...
3066 		 */
3067 		rsize = roundup2(que->num_desc *
3068 		    sizeof(union i40e_rx_desc), DBA_ALIGN);
3069 		rxr = &que->rxr;
3070 		rxr->que = que;
3071 		rxr->tail = I40E_QRX_TAIL(que->me);
3072 
3073 		/* Initialize the RX side lock */
3074 		snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3075 		    device_get_nameunit(dev), que->me);
3076 		mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF);
3077 
3078 		if (i40e_allocate_dma_mem(&pf->hw,
3079 		    &rxr->dma, i40e_mem_reserved, rsize, 4096)) {
3080 			device_printf(dev,
3081 			    "Unable to allocate RX Descriptor memory\n");
3082 			error = ENOMEM;
3083 			goto fail;
3084 		}
3085 		rxr->base = (union i40e_rx_desc *)rxr->dma.va;
3086 		bzero((void *)rxr->base, rsize);
3087 
3088         	/* Allocate receive soft structs for the ring*/
3089 		if (ixl_allocate_rx_data(que)) {
3090 			device_printf(dev,
3091 			    "Critical Failure setting up receive structs\n");
3092 			error = ENOMEM;
3093 			goto fail;
3094 		}
3095 	}
3096 
3097 	return (0);
3098 
3099 fail:
3100 	for (int i = 0; i < vsi->num_queues; i++) {
3101 		que = &vsi->queues[i];
3102 		rxr = &que->rxr;
3103 		txr = &que->txr;
3104 		if (rxr->base)
3105 			i40e_free_dma_mem(&pf->hw, &rxr->dma);
3106 		if (txr->base)
3107 			i40e_free_dma_mem(&pf->hw, &txr->dma);
3108 	}
3109 
3110 early:
3111 	return (error);
3112 }
3113 
3114 /*
3115 ** Provide a update to the queue RX
3116 ** interrupt moderation value.
3117 */
3118 static void
3119 ixl_set_queue_rx_itr(struct ixl_queue *que)
3120 {
3121 	struct ixl_vsi	*vsi = que->vsi;
3122 	struct i40e_hw	*hw = vsi->hw;
3123 	struct rx_ring	*rxr = &que->rxr;
3124 	u16		rx_itr;
3125 	u16		rx_latency = 0;
3126 	int		rx_bytes;
3127 
3128 
3129 	/* Idle, do nothing */
3130 	if (rxr->bytes == 0)
3131 		return;
3132 
3133 	if (ixl_dynamic_rx_itr) {
3134 		rx_bytes = rxr->bytes/rxr->itr;
3135 		rx_itr = rxr->itr;
3136 
3137 		/* Adjust latency range */
3138 		switch (rxr->latency) {
3139 		case IXL_LOW_LATENCY:
3140 			if (rx_bytes > 10) {
3141 				rx_latency = IXL_AVE_LATENCY;
3142 				rx_itr = IXL_ITR_20K;
3143 			}
3144 			break;
3145 		case IXL_AVE_LATENCY:
3146 			if (rx_bytes > 20) {
3147 				rx_latency = IXL_BULK_LATENCY;
3148 				rx_itr = IXL_ITR_8K;
3149 			} else if (rx_bytes <= 10) {
3150 				rx_latency = IXL_LOW_LATENCY;
3151 				rx_itr = IXL_ITR_100K;
3152 			}
3153 			break;
3154 		case IXL_BULK_LATENCY:
3155 			if (rx_bytes <= 20) {
3156 				rx_latency = IXL_AVE_LATENCY;
3157 				rx_itr = IXL_ITR_20K;
3158 			}
3159 			break;
3160        		 }
3161 
3162 		rxr->latency = rx_latency;
3163 
3164 		if (rx_itr != rxr->itr) {
3165 			/* do an exponential smoothing */
3166 			rx_itr = (10 * rx_itr * rxr->itr) /
3167 			    ((9 * rx_itr) + rxr->itr);
3168 			rxr->itr = rx_itr & IXL_MAX_ITR;
3169 			wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3170 			    que->me), rxr->itr);
3171 		}
3172 	} else { /* We may have have toggled to non-dynamic */
3173 		if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC)
3174 			vsi->rx_itr_setting = ixl_rx_itr;
3175 		/* Update the hardware if needed */
3176 		if (rxr->itr != vsi->rx_itr_setting) {
3177 			rxr->itr = vsi->rx_itr_setting;
3178 			wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3179 			    que->me), rxr->itr);
3180 		}
3181 	}
3182 	rxr->bytes = 0;
3183 	rxr->packets = 0;
3184 	return;
3185 }
3186 
3187 
3188 /*
3189 ** Provide a update to the queue TX
3190 ** interrupt moderation value.
3191 */
3192 static void
3193 ixl_set_queue_tx_itr(struct ixl_queue *que)
3194 {
3195 	struct ixl_vsi	*vsi = que->vsi;
3196 	struct i40e_hw	*hw = vsi->hw;
3197 	struct tx_ring	*txr = &que->txr;
3198 	u16		tx_itr;
3199 	u16		tx_latency = 0;
3200 	int		tx_bytes;
3201 
3202 
3203 	/* Idle, do nothing */
3204 	if (txr->bytes == 0)
3205 		return;
3206 
3207 	if (ixl_dynamic_tx_itr) {
3208 		tx_bytes = txr->bytes/txr->itr;
3209 		tx_itr = txr->itr;
3210 
3211 		switch (txr->latency) {
3212 		case IXL_LOW_LATENCY:
3213 			if (tx_bytes > 10) {
3214 				tx_latency = IXL_AVE_LATENCY;
3215 				tx_itr = IXL_ITR_20K;
3216 			}
3217 			break;
3218 		case IXL_AVE_LATENCY:
3219 			if (tx_bytes > 20) {
3220 				tx_latency = IXL_BULK_LATENCY;
3221 				tx_itr = IXL_ITR_8K;
3222 			} else if (tx_bytes <= 10) {
3223 				tx_latency = IXL_LOW_LATENCY;
3224 				tx_itr = IXL_ITR_100K;
3225 			}
3226 			break;
3227 		case IXL_BULK_LATENCY:
3228 			if (tx_bytes <= 20) {
3229 				tx_latency = IXL_AVE_LATENCY;
3230 				tx_itr = IXL_ITR_20K;
3231 			}
3232 			break;
3233 		}
3234 
3235 		txr->latency = tx_latency;
3236 
3237 		if (tx_itr != txr->itr) {
3238        	         /* do an exponential smoothing */
3239 			tx_itr = (10 * tx_itr * txr->itr) /
3240 			    ((9 * tx_itr) + txr->itr);
3241 			txr->itr = tx_itr & IXL_MAX_ITR;
3242 			wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3243 			    que->me), txr->itr);
3244 		}
3245 
3246 	} else { /* We may have have toggled to non-dynamic */
3247 		if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC)
3248 			vsi->tx_itr_setting = ixl_tx_itr;
3249 		/* Update the hardware if needed */
3250 		if (txr->itr != vsi->tx_itr_setting) {
3251 			txr->itr = vsi->tx_itr_setting;
3252 			wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3253 			    que->me), txr->itr);
3254 		}
3255 	}
3256 	txr->bytes = 0;
3257 	txr->packets = 0;
3258 	return;
3259 }
3260 
3261 #define QUEUE_NAME_LEN 32
3262 
3263 static void
3264 ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi,
3265     struct sysctl_ctx_list *ctx, const char *sysctl_name)
3266 {
3267 	struct sysctl_oid *tree;
3268 	struct sysctl_oid_list *child;
3269 	struct sysctl_oid_list *vsi_list;
3270 
3271 	tree = device_get_sysctl_tree(pf->dev);
3272 	child = SYSCTL_CHILDREN(tree);
3273 	vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name,
3274 				   CTLFLAG_RD, NULL, "VSI Number");
3275 	vsi_list = SYSCTL_CHILDREN(vsi->vsi_node);
3276 
3277 	ixl_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats);
3278 }
3279 
3280 static void
3281 ixl_add_hw_stats(struct ixl_pf *pf)
3282 {
3283 	device_t dev = pf->dev;
3284 	struct ixl_vsi *vsi = &pf->vsi;
3285 	struct ixl_queue *queues = vsi->queues;
3286 	struct i40e_hw_port_stats *pf_stats = &pf->stats;
3287 
3288 	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
3289 	struct sysctl_oid *tree = device_get_sysctl_tree(dev);
3290 	struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
3291 	struct sysctl_oid_list *vsi_list;
3292 
3293 	struct sysctl_oid *queue_node;
3294 	struct sysctl_oid_list *queue_list;
3295 
3296 	struct tx_ring *txr;
3297 	struct rx_ring *rxr;
3298 	char queue_namebuf[QUEUE_NAME_LEN];
3299 
3300 	/* Driver statistics */
3301 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
3302 			CTLFLAG_RD, &pf->watchdog_events,
3303 			"Watchdog timeouts");
3304 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq",
3305 			CTLFLAG_RD, &pf->admin_irq,
3306 			"Admin Queue IRQ Handled");
3307 
3308 	ixl_add_vsi_sysctls(pf, &pf->vsi, ctx, "pf");
3309 	vsi_list = SYSCTL_CHILDREN(pf->vsi.vsi_node);
3310 
3311 	/* Queue statistics */
3312 	for (int q = 0; q < vsi->num_queues; q++) {
3313 		snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q);
3314 		queue_node = SYSCTL_ADD_NODE(ctx, vsi_list,
3315 		    OID_AUTO, queue_namebuf, CTLFLAG_RD, NULL, "Queue #");
3316 		queue_list = SYSCTL_CHILDREN(queue_node);
3317 
3318 		txr = &(queues[q].txr);
3319 		rxr = &(queues[q].rxr);
3320 
3321 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed",
3322 				CTLFLAG_RD, &(queues[q].mbuf_defrag_failed),
3323 				"m_defrag() failed");
3324 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "dropped",
3325 				CTLFLAG_RD, &(queues[q].dropped_pkts),
3326 				"Driver dropped packets");
3327 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
3328 				CTLFLAG_RD, &(queues[q].irqs),
3329 				"irqs on this queue");
3330 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx",
3331 				CTLFLAG_RD, &(queues[q].tso),
3332 				"TSO");
3333 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_dma_setup",
3334 				CTLFLAG_RD, &(queues[q].tx_dma_setup),
3335 				"Driver tx dma failure in xmit");
3336 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
3337 				CTLFLAG_RD, &(txr->no_desc),
3338 				"Queue No Descriptor Available");
3339 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
3340 				CTLFLAG_RD, &(txr->total_packets),
3341 				"Queue Packets Transmitted");
3342 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_bytes",
3343 				CTLFLAG_RD, &(txr->tx_bytes),
3344 				"Queue Bytes Transmitted");
3345 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
3346 				CTLFLAG_RD, &(rxr->rx_packets),
3347 				"Queue Packets Received");
3348 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
3349 				CTLFLAG_RD, &(rxr->rx_bytes),
3350 				"Queue Bytes Received");
3351 	}
3352 
3353 	/* MAC stats */
3354 	ixl_add_sysctls_mac_stats(ctx, child, pf_stats);
3355 }
3356 
3357 static void
3358 ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx,
3359 	struct sysctl_oid_list *child,
3360 	struct i40e_eth_stats *eth_stats)
3361 {
3362 	struct ixl_sysctl_info ctls[] =
3363 	{
3364 		{&eth_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"},
3365 		{&eth_stats->rx_unicast, "ucast_pkts_rcvd",
3366 			"Unicast Packets Received"},
3367 		{&eth_stats->rx_multicast, "mcast_pkts_rcvd",
3368 			"Multicast Packets Received"},
3369 		{&eth_stats->rx_broadcast, "bcast_pkts_rcvd",
3370 			"Broadcast Packets Received"},
3371 		{&eth_stats->rx_discards, "rx_discards", "Discarded RX packets"},
3372 		{&eth_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"},
3373 		{&eth_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"},
3374 		{&eth_stats->tx_multicast, "mcast_pkts_txd",
3375 			"Multicast Packets Transmitted"},
3376 		{&eth_stats->tx_broadcast, "bcast_pkts_txd",
3377 			"Broadcast Packets Transmitted"},
3378 		// end
3379 		{0,0,0}
3380 	};
3381 
3382 	struct ixl_sysctl_info *entry = ctls;
3383 	while (entry->stat != 0)
3384 	{
3385 		SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name,
3386 				CTLFLAG_RD, entry->stat,
3387 				entry->description);
3388 		entry++;
3389 	}
3390 }
3391 
3392 static void
3393 ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *ctx,
3394 	struct sysctl_oid_list *child,
3395 	struct i40e_hw_port_stats *stats)
3396 {
3397 	struct sysctl_oid *stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac",
3398 				    CTLFLAG_RD, NULL, "Mac Statistics");
3399 	struct sysctl_oid_list *stat_list = SYSCTL_CHILDREN(stat_node);
3400 
3401 	struct i40e_eth_stats *eth_stats = &stats->eth;
3402 	ixl_add_sysctls_eth_stats(ctx, stat_list, eth_stats);
3403 
3404 	struct ixl_sysctl_info ctls[] =
3405 	{
3406 		{&stats->crc_errors, "crc_errors", "CRC Errors"},
3407 		{&stats->illegal_bytes, "illegal_bytes", "Illegal Byte Errors"},
3408 		{&stats->mac_local_faults, "local_faults", "MAC Local Faults"},
3409 		{&stats->mac_remote_faults, "remote_faults", "MAC Remote Faults"},
3410 		{&stats->rx_length_errors, "rx_length_errors", "Receive Length Errors"},
3411 		/* Packet Reception Stats */
3412 		{&stats->rx_size_64, "rx_frames_64", "64 byte frames received"},
3413 		{&stats->rx_size_127, "rx_frames_65_127", "65-127 byte frames received"},
3414 		{&stats->rx_size_255, "rx_frames_128_255", "128-255 byte frames received"},
3415 		{&stats->rx_size_511, "rx_frames_256_511", "256-511 byte frames received"},
3416 		{&stats->rx_size_1023, "rx_frames_512_1023", "512-1023 byte frames received"},
3417 		{&stats->rx_size_1522, "rx_frames_1024_1522", "1024-1522 byte frames received"},
3418 		{&stats->rx_size_big, "rx_frames_big", "1523-9522 byte frames received"},
3419 		{&stats->rx_undersize, "rx_undersize", "Undersized packets received"},
3420 		{&stats->rx_fragments, "rx_fragmented", "Fragmented packets received"},
3421 		{&stats->rx_oversize, "rx_oversized", "Oversized packets received"},
3422 		{&stats->rx_jabber, "rx_jabber", "Received Jabber"},
3423 		{&stats->checksum_error, "checksum_errors", "Checksum Errors"},
3424 		/* Packet Transmission Stats */
3425 		{&stats->tx_size_64, "tx_frames_64", "64 byte frames transmitted"},
3426 		{&stats->tx_size_127, "tx_frames_65_127", "65-127 byte frames transmitted"},
3427 		{&stats->tx_size_255, "tx_frames_128_255", "128-255 byte frames transmitted"},
3428 		{&stats->tx_size_511, "tx_frames_256_511", "256-511 byte frames transmitted"},
3429 		{&stats->tx_size_1023, "tx_frames_512_1023", "512-1023 byte frames transmitted"},
3430 		{&stats->tx_size_1522, "tx_frames_1024_1522", "1024-1522 byte frames transmitted"},
3431 		{&stats->tx_size_big, "tx_frames_big", "1523-9522 byte frames transmitted"},
3432 		/* Flow control */
3433 		{&stats->link_xon_tx, "xon_txd", "Link XON transmitted"},
3434 		{&stats->link_xon_rx, "xon_recvd", "Link XON received"},
3435 		{&stats->link_xoff_tx, "xoff_txd", "Link XOFF transmitted"},
3436 		{&stats->link_xoff_rx, "xoff_recvd", "Link XOFF received"},
3437 		/* End */
3438 		{0,0,0}
3439 	};
3440 
3441 	struct ixl_sysctl_info *entry = ctls;
3442 	while (entry->stat != 0)
3443 	{
3444 		SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, entry->name,
3445 				CTLFLAG_RD, entry->stat,
3446 				entry->description);
3447 		entry++;
3448 	}
3449 }
3450 
3451 
3452 /*
3453 ** ixl_config_rss - setup RSS
3454 **  - note this is done for the single vsi
3455 */
3456 static void ixl_config_rss(struct ixl_vsi *vsi)
3457 {
3458 	struct ixl_pf	*pf = (struct ixl_pf *)vsi->back;
3459 	struct i40e_hw	*hw = vsi->hw;
3460 	u32		lut = 0;
3461 	u64		set_hena = 0, hena;
3462 	int		i, j, que_id;
3463 #ifdef RSS
3464 	u32		rss_hash_config;
3465 	u32		rss_seed[IXL_KEYSZ];
3466 #else
3467 	u32             rss_seed[IXL_KEYSZ] = {0x41b01687,
3468 			    0x183cfd8c, 0xce880440, 0x580cbc3c,
3469 			    0x35897377, 0x328b25e1, 0x4fa98922,
3470 			    0xb7d90c14, 0xd5bad70d, 0xcd15a2c1};
3471 #endif
3472 
3473 #ifdef RSS
3474         /* Fetch the configured RSS key */
3475         rss_getkey((uint8_t *) &rss_seed);
3476 #endif
3477 
3478 	/* Fill out hash function seed */
3479 	for (i = 0; i < IXL_KEYSZ; i++)
3480                 wr32(hw, I40E_PFQF_HKEY(i), rss_seed[i]);
3481 
3482 	/* Enable PCTYPES for RSS: */
3483 #ifdef RSS
3484 	rss_hash_config = rss_gethashconfig();
3485 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
3486                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER);
3487 	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
3488                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP);
3489 	if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
3490                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP);
3491 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
3492                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER);
3493 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
3494 		set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6);
3495 	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
3496                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP);
3497         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
3498                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP);
3499 #else
3500 	set_hena =
3501 		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) |
3502 		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) |
3503 		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) |
3504 		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) |
3505 		((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) |
3506 		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) |
3507 		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) |
3508 		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) |
3509 		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) |
3510 		((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) |
3511 		((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD);
3512 #endif
3513 	hena = (u64)rd32(hw, I40E_PFQF_HENA(0)) |
3514 	    ((u64)rd32(hw, I40E_PFQF_HENA(1)) << 32);
3515 	hena |= set_hena;
3516 	wr32(hw, I40E_PFQF_HENA(0), (u32)hena);
3517 	wr32(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
3518 
3519 	/* Populate the LUT with max no. of queues in round robin fashion */
3520 	for (i = j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) {
3521 		if (j == vsi->num_queues)
3522 			j = 0;
3523 #ifdef RSS
3524 		/*
3525 		 * Fetch the RSS bucket id for the given indirection entry.
3526 		 * Cap it at the number of configured buckets (which is
3527 		 * num_queues.)
3528 		 */
3529 		que_id = rss_get_indirection_to_bucket(i);
3530 		que_id = que_id % vsi->num_queues;
3531 #else
3532 		que_id = j;
3533 #endif
3534 		/* lut = 4-byte sliding window of 4 lut entries */
3535 		lut = (lut << 8) | (que_id &
3536 		    ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1));
3537 		/* On i = 3, we have 4 entries in lut; write to the register */
3538 		if ((i & 3) == 3)
3539 			wr32(hw, I40E_PFQF_HLUT(i >> 2), lut);
3540 	}
3541 	ixl_flush(hw);
3542 }
3543 
3544 
3545 /*
3546 ** This routine is run via an vlan config EVENT,
3547 ** it enables us to use the HW Filter table since
3548 ** we can get the vlan id. This just creates the
3549 ** entry in the soft version of the VFTA, init will
3550 ** repopulate the real table.
3551 */
3552 static void
3553 ixl_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3554 {
3555 	struct ixl_vsi	*vsi = ifp->if_softc;
3556 	struct i40e_hw	*hw = vsi->hw;
3557 	struct ixl_pf	*pf = (struct ixl_pf *)vsi->back;
3558 
3559 	if (ifp->if_softc !=  arg)   /* Not our event */
3560 		return;
3561 
3562 	if ((vtag == 0) || (vtag > 4095))	/* Invalid */
3563 		return;
3564 
3565 	IXL_PF_LOCK(pf);
3566 	++vsi->num_vlans;
3567 	ixl_add_filter(vsi, hw->mac.addr, vtag);
3568 	IXL_PF_UNLOCK(pf);
3569 }
3570 
3571 /*
3572 ** This routine is run via an vlan
3573 ** unconfig EVENT, remove our entry
3574 ** in the soft vfta.
3575 */
3576 static void
3577 ixl_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3578 {
3579 	struct ixl_vsi	*vsi = ifp->if_softc;
3580 	struct i40e_hw	*hw = vsi->hw;
3581 	struct ixl_pf	*pf = (struct ixl_pf *)vsi->back;
3582 
3583 	if (ifp->if_softc !=  arg)
3584 		return;
3585 
3586 	if ((vtag == 0) || (vtag > 4095))	/* Invalid */
3587 		return;
3588 
3589 	IXL_PF_LOCK(pf);
3590 	--vsi->num_vlans;
3591 	ixl_del_filter(vsi, hw->mac.addr, vtag);
3592 	IXL_PF_UNLOCK(pf);
3593 }
3594 
3595 /*
3596 ** This routine updates vlan filters, called by init
3597 ** it scans the filter table and then updates the hw
3598 ** after a soft reset.
3599 */
3600 static void
3601 ixl_setup_vlan_filters(struct ixl_vsi *vsi)
3602 {
3603 	struct ixl_mac_filter	*f;
3604 	int			cnt = 0, flags;
3605 
3606 	if (vsi->num_vlans == 0)
3607 		return;
3608 	/*
3609 	** Scan the filter list for vlan entries,
3610 	** mark them for addition and then call
3611 	** for the AQ update.
3612 	*/
3613 	SLIST_FOREACH(f, &vsi->ftl, next) {
3614 		if (f->flags & IXL_FILTER_VLAN) {
3615 			f->flags |=
3616 			    (IXL_FILTER_ADD |
3617 			    IXL_FILTER_USED);
3618 			cnt++;
3619 		}
3620 	}
3621 	if (cnt == 0) {
3622 		printf("setup vlan: no filters found!\n");
3623 		return;
3624 	}
3625 	flags = IXL_FILTER_VLAN;
3626 	flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
3627 	ixl_add_hw_filters(vsi, flags, cnt);
3628 	return;
3629 }
3630 
3631 /*
3632 ** Initialize filter list and add filters that the hardware
3633 ** needs to know about.
3634 */
3635 static void
3636 ixl_init_filters(struct ixl_vsi *vsi)
3637 {
3638 	/* Add broadcast address */
3639 	ixl_add_filter(vsi, ixl_bcast_addr, IXL_VLAN_ANY);
3640 }
3641 
3642 /*
3643 ** This routine adds mulicast filters
3644 */
3645 static void
3646 ixl_add_mc_filter(struct ixl_vsi *vsi, u8 *macaddr)
3647 {
3648 	struct ixl_mac_filter *f;
3649 
3650 	/* Does one already exist */
3651 	f = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
3652 	if (f != NULL)
3653 		return;
3654 
3655 	f = ixl_get_filter(vsi);
3656 	if (f == NULL) {
3657 		printf("WARNING: no filter available!!\n");
3658 		return;
3659 	}
3660 	bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
3661 	f->vlan = IXL_VLAN_ANY;
3662 	f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED
3663 	    | IXL_FILTER_MC);
3664 
3665 	return;
3666 }
3667 
3668 static void
3669 ixl_reconfigure_filters(struct ixl_vsi *vsi)
3670 {
3671 
3672 	ixl_add_hw_filters(vsi, IXL_FILTER_USED, vsi->num_macs);
3673 }
3674 
3675 /*
3676 ** This routine adds macvlan filters
3677 */
3678 static void
3679 ixl_add_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
3680 {
3681 	struct ixl_mac_filter	*f, *tmp;
3682 	struct ixl_pf		*pf;
3683 	device_t		dev;
3684 
3685 	DEBUGOUT("ixl_add_filter: begin");
3686 
3687 	pf = vsi->back;
3688 	dev = pf->dev;
3689 
3690 	/* Does one already exist */
3691 	f = ixl_find_filter(vsi, macaddr, vlan);
3692 	if (f != NULL)
3693 		return;
3694 	/*
3695 	** Is this the first vlan being registered, if so we
3696 	** need to remove the ANY filter that indicates we are
3697 	** not in a vlan, and replace that with a 0 filter.
3698 	*/
3699 	if ((vlan != IXL_VLAN_ANY) && (vsi->num_vlans == 1)) {
3700 		tmp = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
3701 		if (tmp != NULL) {
3702 			ixl_del_filter(vsi, macaddr, IXL_VLAN_ANY);
3703 			ixl_add_filter(vsi, macaddr, 0);
3704 		}
3705 	}
3706 
3707 	f = ixl_get_filter(vsi);
3708 	if (f == NULL) {
3709 		device_printf(dev, "WARNING: no filter available!!\n");
3710 		return;
3711 	}
3712 	bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
3713 	f->vlan = vlan;
3714 	f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
3715 	if (f->vlan != IXL_VLAN_ANY)
3716 		f->flags |= IXL_FILTER_VLAN;
3717 	else
3718 		vsi->num_macs++;
3719 
3720 	ixl_add_hw_filters(vsi, f->flags, 1);
3721 	return;
3722 }
3723 
3724 static void
3725 ixl_del_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
3726 {
3727 	struct ixl_mac_filter *f;
3728 
3729 	f = ixl_find_filter(vsi, macaddr, vlan);
3730 	if (f == NULL)
3731 		return;
3732 
3733 	f->flags |= IXL_FILTER_DEL;
3734 	ixl_del_hw_filters(vsi, 1);
3735 	vsi->num_macs--;
3736 
3737 	/* Check if this is the last vlan removal */
3738 	if (vlan != IXL_VLAN_ANY && vsi->num_vlans == 0) {
3739 		/* Switch back to a non-vlan filter */
3740 		ixl_del_filter(vsi, macaddr, 0);
3741 		ixl_add_filter(vsi, macaddr, IXL_VLAN_ANY);
3742 	}
3743 	return;
3744 }
3745 
3746 /*
3747 ** Find the filter with both matching mac addr and vlan id
3748 */
3749 static struct ixl_mac_filter *
3750 ixl_find_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
3751 {
3752 	struct ixl_mac_filter	*f;
3753 	bool			match = FALSE;
3754 
3755 	SLIST_FOREACH(f, &vsi->ftl, next) {
3756 		if (!cmp_etheraddr(f->macaddr, macaddr))
3757 			continue;
3758 		if (f->vlan == vlan) {
3759 			match = TRUE;
3760 			break;
3761 		}
3762 	}
3763 
3764 	if (!match)
3765 		f = NULL;
3766 	return (f);
3767 }
3768 
3769 /*
3770 ** This routine takes additions to the vsi filter
3771 ** table and creates an Admin Queue call to create
3772 ** the filters in the hardware.
3773 */
3774 static void
3775 ixl_add_hw_filters(struct ixl_vsi *vsi, int flags, int cnt)
3776 {
3777 	struct i40e_aqc_add_macvlan_element_data *a, *b;
3778 	struct ixl_mac_filter	*f;
3779 	struct ixl_pf		*pf;
3780 	struct i40e_hw		*hw;
3781 	device_t		dev;
3782 	int			err, j = 0;
3783 
3784 	pf = vsi->back;
3785 	dev = pf->dev;
3786 	hw = &pf->hw;
3787 	IXL_PF_LOCK_ASSERT(pf);
3788 
3789 	a = malloc(sizeof(struct i40e_aqc_add_macvlan_element_data) * cnt,
3790 	    M_DEVBUF, M_NOWAIT | M_ZERO);
3791 	if (a == NULL) {
3792 		device_printf(dev, "add_hw_filters failed to get memory\n");
3793 		return;
3794 	}
3795 
3796 	/*
3797 	** Scan the filter list, each time we find one
3798 	** we add it to the admin queue array and turn off
3799 	** the add bit.
3800 	*/
3801 	SLIST_FOREACH(f, &vsi->ftl, next) {
3802 		if (f->flags == flags) {
3803 			b = &a[j]; // a pox on fvl long names :)
3804 			bcopy(f->macaddr, b->mac_addr, ETHER_ADDR_LEN);
3805 			if (f->vlan == IXL_VLAN_ANY) {
3806 				b->vlan_tag = 0;
3807 				b->flags = I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
3808 			} else {
3809 				b->vlan_tag = f->vlan;
3810 				b->flags = 0;
3811 			}
3812 			b->flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
3813 			f->flags &= ~IXL_FILTER_ADD;
3814 			j++;
3815 		}
3816 		if (j == cnt)
3817 			break;
3818 	}
3819 	if (j > 0) {
3820 		err = i40e_aq_add_macvlan(hw, vsi->seid, a, j, NULL);
3821 		if (err)
3822 			device_printf(dev, "aq_add_macvlan err %d, "
3823 			    "aq_error %d\n", err, hw->aq.asq_last_status);
3824 		else
3825 			vsi->hw_filters_add += j;
3826 	}
3827 	free(a, M_DEVBUF);
3828 	return;
3829 }
3830 
3831 /*
3832 ** This routine takes removals in the vsi filter
3833 ** table and creates an Admin Queue call to delete
3834 ** the filters in the hardware.
3835 */
3836 static void
3837 ixl_del_hw_filters(struct ixl_vsi *vsi, int cnt)
3838 {
3839 	struct i40e_aqc_remove_macvlan_element_data *d, *e;
3840 	struct ixl_pf		*pf;
3841 	struct i40e_hw		*hw;
3842 	device_t		dev;
3843 	struct ixl_mac_filter	*f, *f_temp;
3844 	int			err, j = 0;
3845 
3846 	DEBUGOUT("ixl_del_hw_filters: begin\n");
3847 
3848 	pf = vsi->back;
3849 	hw = &pf->hw;
3850 	dev = pf->dev;
3851 
3852 	d = malloc(sizeof(struct i40e_aqc_remove_macvlan_element_data) * cnt,
3853 	    M_DEVBUF, M_NOWAIT | M_ZERO);
3854 	if (d == NULL) {
3855 		printf("del hw filter failed to get memory\n");
3856 		return;
3857 	}
3858 
3859 	SLIST_FOREACH_SAFE(f, &vsi->ftl, next, f_temp) {
3860 		if (f->flags & IXL_FILTER_DEL) {
3861 			e = &d[j]; // a pox on fvl long names :)
3862 			bcopy(f->macaddr, e->mac_addr, ETHER_ADDR_LEN);
3863 			e->vlan_tag = (f->vlan == IXL_VLAN_ANY ? 0 : f->vlan);
3864 			e->flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
3865 			/* delete entry from vsi list */
3866 			SLIST_REMOVE(&vsi->ftl, f, ixl_mac_filter, next);
3867 			free(f, M_DEVBUF);
3868 			j++;
3869 		}
3870 		if (j == cnt)
3871 			break;
3872 	}
3873 	if (j > 0) {
3874 		err = i40e_aq_remove_macvlan(hw, vsi->seid, d, j, NULL);
3875 		/* NOTE: returns ENOENT every time but seems to work fine,
3876 		   so we'll ignore that specific error. */
3877 		// TODO: Does this still occur on current firmwares?
3878 		if (err && hw->aq.asq_last_status != I40E_AQ_RC_ENOENT) {
3879 			int sc = 0;
3880 			for (int i = 0; i < j; i++)
3881 				sc += (!d[i].error_code);
3882 			vsi->hw_filters_del += sc;
3883 			device_printf(dev,
3884 			    "Failed to remove %d/%d filters, aq error %d\n",
3885 			    j - sc, j, hw->aq.asq_last_status);
3886 		} else
3887 			vsi->hw_filters_del += j;
3888 	}
3889 	free(d, M_DEVBUF);
3890 
3891 	DEBUGOUT("ixl_del_hw_filters: end\n");
3892 	return;
3893 }
3894 
3895 static int
3896 ixl_enable_rings(struct ixl_vsi *vsi)
3897 {
3898 	struct ixl_pf	*pf = vsi->back;
3899 	struct i40e_hw	*hw = &pf->hw;
3900 	int		index, error;
3901 	u32		reg;
3902 
3903 	error = 0;
3904 	for (int i = 0; i < vsi->num_queues; i++) {
3905 		index = vsi->first_queue + i;
3906 		i40e_pre_tx_queue_cfg(hw, index, TRUE);
3907 
3908 		reg = rd32(hw, I40E_QTX_ENA(index));
3909 		reg |= I40E_QTX_ENA_QENA_REQ_MASK |
3910 		    I40E_QTX_ENA_QENA_STAT_MASK;
3911 		wr32(hw, I40E_QTX_ENA(index), reg);
3912 		/* Verify the enable took */
3913 		for (int j = 0; j < 10; j++) {
3914 			reg = rd32(hw, I40E_QTX_ENA(index));
3915 			if (reg & I40E_QTX_ENA_QENA_STAT_MASK)
3916 				break;
3917 			i40e_msec_delay(10);
3918 		}
3919 		if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) {
3920 			device_printf(pf->dev, "TX queue %d disabled!\n",
3921 			    index);
3922 			error = ETIMEDOUT;
3923 		}
3924 
3925 		reg = rd32(hw, I40E_QRX_ENA(index));
3926 		reg |= I40E_QRX_ENA_QENA_REQ_MASK |
3927 		    I40E_QRX_ENA_QENA_STAT_MASK;
3928 		wr32(hw, I40E_QRX_ENA(index), reg);
3929 		/* Verify the enable took */
3930 		for (int j = 0; j < 10; j++) {
3931 			reg = rd32(hw, I40E_QRX_ENA(index));
3932 			if (reg & I40E_QRX_ENA_QENA_STAT_MASK)
3933 				break;
3934 			i40e_msec_delay(10);
3935 		}
3936 		if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) {
3937 			device_printf(pf->dev, "RX queue %d disabled!\n",
3938 			    index);
3939 			error = ETIMEDOUT;
3940 		}
3941 	}
3942 
3943 	return (error);
3944 }
3945 
3946 static int
3947 ixl_disable_rings(struct ixl_vsi *vsi)
3948 {
3949 	struct ixl_pf	*pf = vsi->back;
3950 	struct i40e_hw	*hw = &pf->hw;
3951 	int		index, error;
3952 	u32		reg;
3953 
3954 	error = 0;
3955 	for (int i = 0; i < vsi->num_queues; i++) {
3956 		index = vsi->first_queue + i;
3957 
3958 		i40e_pre_tx_queue_cfg(hw, index, FALSE);
3959 		i40e_usec_delay(500);
3960 
3961 		reg = rd32(hw, I40E_QTX_ENA(index));
3962 		reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
3963 		wr32(hw, I40E_QTX_ENA(index), reg);
3964 		/* Verify the disable took */
3965 		for (int j = 0; j < 10; j++) {
3966 			reg = rd32(hw, I40E_QTX_ENA(index));
3967 			if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
3968 				break;
3969 			i40e_msec_delay(10);
3970 		}
3971 		if (reg & I40E_QTX_ENA_QENA_STAT_MASK) {
3972 			device_printf(pf->dev, "TX queue %d still enabled!\n",
3973 			    index);
3974 			error = ETIMEDOUT;
3975 		}
3976 
3977 		reg = rd32(hw, I40E_QRX_ENA(index));
3978 		reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
3979 		wr32(hw, I40E_QRX_ENA(index), reg);
3980 		/* Verify the disable took */
3981 		for (int j = 0; j < 10; j++) {
3982 			reg = rd32(hw, I40E_QRX_ENA(index));
3983 			if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
3984 				break;
3985 			i40e_msec_delay(10);
3986 		}
3987 		if (reg & I40E_QRX_ENA_QENA_STAT_MASK) {
3988 			device_printf(pf->dev, "RX queue %d still enabled!\n",
3989 			    index);
3990 			error = ETIMEDOUT;
3991 		}
3992 	}
3993 
3994 	return (error);
3995 }
3996 
3997 /**
3998  * ixl_handle_mdd_event
3999  *
4000  * Called from interrupt handler to identify possibly malicious vfs
4001  * (But also detects events from the PF, as well)
4002  **/
4003 static void ixl_handle_mdd_event(struct ixl_pf *pf)
4004 {
4005 	struct i40e_hw *hw = &pf->hw;
4006 	device_t dev = pf->dev;
4007 	bool mdd_detected = false;
4008 	bool pf_mdd_detected = false;
4009 	u32 reg;
4010 
4011 	/* find what triggered the MDD event */
4012 	reg = rd32(hw, I40E_GL_MDET_TX);
4013 	if (reg & I40E_GL_MDET_TX_VALID_MASK) {
4014 		u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
4015 				I40E_GL_MDET_TX_PF_NUM_SHIFT;
4016 		u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
4017 				I40E_GL_MDET_TX_EVENT_SHIFT;
4018 		u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
4019 				I40E_GL_MDET_TX_QUEUE_SHIFT;
4020 		device_printf(dev,
4021 			 "Malicious Driver Detection event 0x%02x"
4022 			 " on TX queue %d pf number 0x%02x\n",
4023 			 event, queue, pf_num);
4024 		wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
4025 		mdd_detected = true;
4026 	}
4027 	reg = rd32(hw, I40E_GL_MDET_RX);
4028 	if (reg & I40E_GL_MDET_RX_VALID_MASK) {
4029 		u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
4030 				I40E_GL_MDET_RX_FUNCTION_SHIFT;
4031 		u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
4032 				I40E_GL_MDET_RX_EVENT_SHIFT;
4033 		u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
4034 				I40E_GL_MDET_RX_QUEUE_SHIFT;
4035 		device_printf(dev,
4036 			 "Malicious Driver Detection event 0x%02x"
4037 			 " on RX queue %d of function 0x%02x\n",
4038 			 event, queue, func);
4039 		wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
4040 		mdd_detected = true;
4041 	}
4042 
4043 	if (mdd_detected) {
4044 		reg = rd32(hw, I40E_PF_MDET_TX);
4045 		if (reg & I40E_PF_MDET_TX_VALID_MASK) {
4046 			wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
4047 			device_printf(dev,
4048 				 "MDD TX event is for this function 0x%08x",
4049 				 reg);
4050 			pf_mdd_detected = true;
4051 		}
4052 		reg = rd32(hw, I40E_PF_MDET_RX);
4053 		if (reg & I40E_PF_MDET_RX_VALID_MASK) {
4054 			wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
4055 			device_printf(dev,
4056 				 "MDD RX event is for this function 0x%08x",
4057 				 reg);
4058 			pf_mdd_detected = true;
4059 		}
4060 	}
4061 
4062 	/* re-enable mdd interrupt cause */
4063 	reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4064 	reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
4065 	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4066 	ixl_flush(hw);
4067 }
4068 
4069 static void
4070 ixl_enable_intr(struct ixl_vsi *vsi)
4071 {
4072 	struct i40e_hw		*hw = vsi->hw;
4073 	struct ixl_queue	*que = vsi->queues;
4074 
4075 	if (ixl_enable_msix) {
4076 		ixl_enable_adminq(hw);
4077 		for (int i = 0; i < vsi->num_queues; i++, que++)
4078 			ixl_enable_queue(hw, que->me);
4079 	} else
4080 		ixl_enable_legacy(hw);
4081 }
4082 
4083 static void
4084 ixl_disable_rings_intr(struct ixl_vsi *vsi)
4085 {
4086 	struct i40e_hw		*hw = vsi->hw;
4087 	struct ixl_queue	*que = vsi->queues;
4088 
4089 	for (int i = 0; i < vsi->num_queues; i++, que++)
4090 		ixl_disable_queue(hw, que->me);
4091 }
4092 
4093 static void
4094 ixl_disable_intr(struct ixl_vsi *vsi)
4095 {
4096 	struct i40e_hw		*hw = vsi->hw;
4097 
4098 	if (ixl_enable_msix)
4099 		ixl_disable_adminq(hw);
4100 	else
4101 		ixl_disable_legacy(hw);
4102 }
4103 
4104 static void
4105 ixl_enable_adminq(struct i40e_hw *hw)
4106 {
4107 	u32		reg;
4108 
4109 	reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4110 	    I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4111 	    (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4112 	wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4113 	ixl_flush(hw);
4114 	return;
4115 }
4116 
4117 static void
4118 ixl_disable_adminq(struct i40e_hw *hw)
4119 {
4120 	u32		reg;
4121 
4122 	reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4123 	wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4124 
4125 	return;
4126 }
4127 
4128 static void
4129 ixl_enable_queue(struct i40e_hw *hw, int id)
4130 {
4131 	u32		reg;
4132 
4133 	reg = I40E_PFINT_DYN_CTLN_INTENA_MASK |
4134 	    I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
4135 	    (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
4136 	wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4137 }
4138 
4139 static void
4140 ixl_disable_queue(struct i40e_hw *hw, int id)
4141 {
4142 	u32		reg;
4143 
4144 	reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT;
4145 	wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4146 
4147 	return;
4148 }
4149 
4150 static void
4151 ixl_enable_legacy(struct i40e_hw *hw)
4152 {
4153 	u32		reg;
4154 	reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4155 	    I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4156 	    (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4157 	wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4158 }
4159 
4160 static void
4161 ixl_disable_legacy(struct i40e_hw *hw)
4162 {
4163 	u32		reg;
4164 
4165 	reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4166 	wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4167 
4168 	return;
4169 }
4170 
4171 static void
4172 ixl_update_stats_counters(struct ixl_pf *pf)
4173 {
4174 	struct i40e_hw	*hw = &pf->hw;
4175 	struct ixl_vsi	*vsi = &pf->vsi;
4176 	struct ixl_vf	*vf;
4177 
4178 	struct i40e_hw_port_stats *nsd = &pf->stats;
4179 	struct i40e_hw_port_stats *osd = &pf->stats_offsets;
4180 
4181 	/* Update hw stats */
4182 	ixl_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
4183 			   pf->stat_offsets_loaded,
4184 			   &osd->crc_errors, &nsd->crc_errors);
4185 	ixl_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
4186 			   pf->stat_offsets_loaded,
4187 			   &osd->illegal_bytes, &nsd->illegal_bytes);
4188 	ixl_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
4189 			   I40E_GLPRT_GORCL(hw->port),
4190 			   pf->stat_offsets_loaded,
4191 			   &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
4192 	ixl_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
4193 			   I40E_GLPRT_GOTCL(hw->port),
4194 			   pf->stat_offsets_loaded,
4195 			   &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
4196 	ixl_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
4197 			   pf->stat_offsets_loaded,
4198 			   &osd->eth.rx_discards,
4199 			   &nsd->eth.rx_discards);
4200 	ixl_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
4201 			   I40E_GLPRT_UPRCL(hw->port),
4202 			   pf->stat_offsets_loaded,
4203 			   &osd->eth.rx_unicast,
4204 			   &nsd->eth.rx_unicast);
4205 	ixl_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
4206 			   I40E_GLPRT_UPTCL(hw->port),
4207 			   pf->stat_offsets_loaded,
4208 			   &osd->eth.tx_unicast,
4209 			   &nsd->eth.tx_unicast);
4210 	ixl_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
4211 			   I40E_GLPRT_MPRCL(hw->port),
4212 			   pf->stat_offsets_loaded,
4213 			   &osd->eth.rx_multicast,
4214 			   &nsd->eth.rx_multicast);
4215 	ixl_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
4216 			   I40E_GLPRT_MPTCL(hw->port),
4217 			   pf->stat_offsets_loaded,
4218 			   &osd->eth.tx_multicast,
4219 			   &nsd->eth.tx_multicast);
4220 	ixl_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
4221 			   I40E_GLPRT_BPRCL(hw->port),
4222 			   pf->stat_offsets_loaded,
4223 			   &osd->eth.rx_broadcast,
4224 			   &nsd->eth.rx_broadcast);
4225 	ixl_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
4226 			   I40E_GLPRT_BPTCL(hw->port),
4227 			   pf->stat_offsets_loaded,
4228 			   &osd->eth.tx_broadcast,
4229 			   &nsd->eth.tx_broadcast);
4230 
4231 	ixl_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
4232 			   pf->stat_offsets_loaded,
4233 			   &osd->tx_dropped_link_down,
4234 			   &nsd->tx_dropped_link_down);
4235 	ixl_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
4236 			   pf->stat_offsets_loaded,
4237 			   &osd->mac_local_faults,
4238 			   &nsd->mac_local_faults);
4239 	ixl_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
4240 			   pf->stat_offsets_loaded,
4241 			   &osd->mac_remote_faults,
4242 			   &nsd->mac_remote_faults);
4243 	ixl_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
4244 			   pf->stat_offsets_loaded,
4245 			   &osd->rx_length_errors,
4246 			   &nsd->rx_length_errors);
4247 
4248 	/* Flow control (LFC) stats */
4249 	ixl_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
4250 			   pf->stat_offsets_loaded,
4251 			   &osd->link_xon_rx, &nsd->link_xon_rx);
4252 	ixl_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
4253 			   pf->stat_offsets_loaded,
4254 			   &osd->link_xon_tx, &nsd->link_xon_tx);
4255 	ixl_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
4256 			   pf->stat_offsets_loaded,
4257 			   &osd->link_xoff_rx, &nsd->link_xoff_rx);
4258 	ixl_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
4259 			   pf->stat_offsets_loaded,
4260 			   &osd->link_xoff_tx, &nsd->link_xoff_tx);
4261 
4262 	/* Packet size stats rx */
4263 	ixl_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
4264 			   I40E_GLPRT_PRC64L(hw->port),
4265 			   pf->stat_offsets_loaded,
4266 			   &osd->rx_size_64, &nsd->rx_size_64);
4267 	ixl_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
4268 			   I40E_GLPRT_PRC127L(hw->port),
4269 			   pf->stat_offsets_loaded,
4270 			   &osd->rx_size_127, &nsd->rx_size_127);
4271 	ixl_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
4272 			   I40E_GLPRT_PRC255L(hw->port),
4273 			   pf->stat_offsets_loaded,
4274 			   &osd->rx_size_255, &nsd->rx_size_255);
4275 	ixl_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
4276 			   I40E_GLPRT_PRC511L(hw->port),
4277 			   pf->stat_offsets_loaded,
4278 			   &osd->rx_size_511, &nsd->rx_size_511);
4279 	ixl_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
4280 			   I40E_GLPRT_PRC1023L(hw->port),
4281 			   pf->stat_offsets_loaded,
4282 			   &osd->rx_size_1023, &nsd->rx_size_1023);
4283 	ixl_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
4284 			   I40E_GLPRT_PRC1522L(hw->port),
4285 			   pf->stat_offsets_loaded,
4286 			   &osd->rx_size_1522, &nsd->rx_size_1522);
4287 	ixl_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
4288 			   I40E_GLPRT_PRC9522L(hw->port),
4289 			   pf->stat_offsets_loaded,
4290 			   &osd->rx_size_big, &nsd->rx_size_big);
4291 
4292 	/* Packet size stats tx */
4293 	ixl_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
4294 			   I40E_GLPRT_PTC64L(hw->port),
4295 			   pf->stat_offsets_loaded,
4296 			   &osd->tx_size_64, &nsd->tx_size_64);
4297 	ixl_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
4298 			   I40E_GLPRT_PTC127L(hw->port),
4299 			   pf->stat_offsets_loaded,
4300 			   &osd->tx_size_127, &nsd->tx_size_127);
4301 	ixl_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
4302 			   I40E_GLPRT_PTC255L(hw->port),
4303 			   pf->stat_offsets_loaded,
4304 			   &osd->tx_size_255, &nsd->tx_size_255);
4305 	ixl_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
4306 			   I40E_GLPRT_PTC511L(hw->port),
4307 			   pf->stat_offsets_loaded,
4308 			   &osd->tx_size_511, &nsd->tx_size_511);
4309 	ixl_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
4310 			   I40E_GLPRT_PTC1023L(hw->port),
4311 			   pf->stat_offsets_loaded,
4312 			   &osd->tx_size_1023, &nsd->tx_size_1023);
4313 	ixl_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
4314 			   I40E_GLPRT_PTC1522L(hw->port),
4315 			   pf->stat_offsets_loaded,
4316 			   &osd->tx_size_1522, &nsd->tx_size_1522);
4317 	ixl_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
4318 			   I40E_GLPRT_PTC9522L(hw->port),
4319 			   pf->stat_offsets_loaded,
4320 			   &osd->tx_size_big, &nsd->tx_size_big);
4321 
4322 	ixl_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
4323 			   pf->stat_offsets_loaded,
4324 			   &osd->rx_undersize, &nsd->rx_undersize);
4325 	ixl_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
4326 			   pf->stat_offsets_loaded,
4327 			   &osd->rx_fragments, &nsd->rx_fragments);
4328 	ixl_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
4329 			   pf->stat_offsets_loaded,
4330 			   &osd->rx_oversize, &nsd->rx_oversize);
4331 	ixl_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
4332 			   pf->stat_offsets_loaded,
4333 			   &osd->rx_jabber, &nsd->rx_jabber);
4334 	pf->stat_offsets_loaded = true;
4335 	/* End hw stats */
4336 
4337 	/* Update vsi stats */
4338 	ixl_update_vsi_stats(vsi);
4339 
4340 	for (int i = 0; i < pf->num_vfs; i++) {
4341 		vf = &pf->vfs[i];
4342 		if (vf->vf_flags & VF_FLAG_ENABLED)
4343 			ixl_update_eth_stats(&pf->vfs[i].vsi);
4344 	}
4345 }
4346 
4347 /*
4348 ** Tasklet handler for MSIX Adminq interrupts
4349 **  - do outside interrupt since it might sleep
4350 */
4351 static void
4352 ixl_do_adminq(void *context, int pending)
4353 {
4354 	struct ixl_pf			*pf = context;
4355 	struct i40e_hw			*hw = &pf->hw;
4356 	struct ixl_vsi			*vsi = &pf->vsi;
4357 	struct i40e_arq_event_info	event;
4358 	i40e_status			ret;
4359 	u32				reg, loop = 0;
4360 	u16				opcode, result;
4361 
4362 	event.buf_len = IXL_AQ_BUF_SZ;
4363 	event.msg_buf = malloc(event.buf_len,
4364 	    M_DEVBUF, M_NOWAIT | M_ZERO);
4365 	if (!event.msg_buf) {
4366 		printf("Unable to allocate adminq memory\n");
4367 		return;
4368 	}
4369 
4370 	IXL_PF_LOCK(pf);
4371 	/* clean and process any events */
4372 	do {
4373 		ret = i40e_clean_arq_element(hw, &event, &result);
4374 		if (ret)
4375 			break;
4376 		opcode = LE16_TO_CPU(event.desc.opcode);
4377 		switch (opcode) {
4378 		case i40e_aqc_opc_get_link_status:
4379 			ixl_link_event(pf, &event);
4380 			ixl_update_link_status(pf);
4381 			break;
4382 		case i40e_aqc_opc_send_msg_to_pf:
4383 #ifdef PCI_IOV
4384 			ixl_handle_vf_msg(pf, &event);
4385 #endif
4386 			break;
4387 		case i40e_aqc_opc_event_lan_overflow:
4388 			break;
4389 		default:
4390 #ifdef IXL_DEBUG
4391 			printf("AdminQ unknown event %x\n", opcode);
4392 #endif
4393 			break;
4394 		}
4395 
4396 	} while (result && (loop++ < IXL_ADM_LIMIT));
4397 
4398 	reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4399 	reg |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
4400 	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4401 	free(event.msg_buf, M_DEVBUF);
4402 
4403 	/*
4404 	 * If there are still messages to process, reschedule ourselves.
4405 	 * Otherwise, re-enable our interrupt and go to sleep.
4406 	 */
4407 	if (result > 0)
4408 		taskqueue_enqueue(pf->tq, &pf->adminq);
4409 	else
4410 		ixl_enable_intr(vsi);
4411 
4412 	IXL_PF_UNLOCK(pf);
4413 }
4414 
4415 #ifdef IXL_DEBUG_SYSCTL
4416 static int
4417 ixl_debug_info(SYSCTL_HANDLER_ARGS)
4418 {
4419 	struct ixl_pf	*pf;
4420 	int		error, input = 0;
4421 
4422 	error = sysctl_handle_int(oidp, &input, 0, req);
4423 
4424 	if (error || !req->newptr)
4425 		return (error);
4426 
4427 	if (input == 1) {
4428 		pf = (struct ixl_pf *)arg1;
4429 		ixl_print_debug_info(pf);
4430 	}
4431 
4432 	return (error);
4433 }
4434 
4435 static void
4436 ixl_print_debug_info(struct ixl_pf *pf)
4437 {
4438 	struct i40e_hw		*hw = &pf->hw;
4439 	struct ixl_vsi		*vsi = &pf->vsi;
4440 	struct ixl_queue	*que = vsi->queues;
4441 	struct rx_ring		*rxr = &que->rxr;
4442 	struct tx_ring		*txr = &que->txr;
4443 	u32			reg;
4444 
4445 
4446 	printf("Queue irqs = %jx\n", (uintmax_t)que->irqs);
4447 	printf("AdminQ irqs = %jx\n", (uintmax_t)pf->admin_irq);
4448 	printf("RX next check = %x\n", rxr->next_check);
4449 	printf("RX not ready = %jx\n", (uintmax_t)rxr->not_done);
4450 	printf("RX packets = %jx\n", (uintmax_t)rxr->rx_packets);
4451 	printf("TX desc avail = %x\n", txr->avail);
4452 
4453 	reg = rd32(hw, I40E_GLV_GORCL(0xc));
4454 	 printf("RX Bytes = %x\n", reg);
4455 	reg = rd32(hw, I40E_GLPRT_GORCL(hw->port));
4456 	 printf("Port RX Bytes = %x\n", reg);
4457 	reg = rd32(hw, I40E_GLV_RDPC(0xc));
4458 	 printf("RX discard = %x\n", reg);
4459 	reg = rd32(hw, I40E_GLPRT_RDPC(hw->port));
4460 	 printf("Port RX discard = %x\n", reg);
4461 
4462 	reg = rd32(hw, I40E_GLV_TEPC(0xc));
4463 	 printf("TX errors = %x\n", reg);
4464 	reg = rd32(hw, I40E_GLV_GOTCL(0xc));
4465 	 printf("TX Bytes = %x\n", reg);
4466 
4467 	reg = rd32(hw, I40E_GLPRT_RUC(hw->port));
4468 	 printf("RX undersize = %x\n", reg);
4469 	reg = rd32(hw, I40E_GLPRT_RFC(hw->port));
4470 	 printf("RX fragments = %x\n", reg);
4471 	reg = rd32(hw, I40E_GLPRT_ROC(hw->port));
4472 	 printf("RX oversize = %x\n", reg);
4473 	reg = rd32(hw, I40E_GLPRT_RLEC(hw->port));
4474 	 printf("RX length error = %x\n", reg);
4475 	reg = rd32(hw, I40E_GLPRT_MRFC(hw->port));
4476 	 printf("mac remote fault = %x\n", reg);
4477 	reg = rd32(hw, I40E_GLPRT_MLFC(hw->port));
4478 	 printf("mac local fault = %x\n", reg);
4479 }
4480 #endif
4481 
4482 /**
4483  * Update VSI-specific ethernet statistics counters.
4484  **/
4485 void ixl_update_eth_stats(struct ixl_vsi *vsi)
4486 {
4487 	struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
4488 	struct i40e_hw *hw = &pf->hw;
4489 	struct i40e_eth_stats *es;
4490 	struct i40e_eth_stats *oes;
4491 	struct i40e_hw_port_stats *nsd;
4492 	u16 stat_idx = vsi->info.stat_counter_idx;
4493 
4494 	es = &vsi->eth_stats;
4495 	oes = &vsi->eth_stats_offsets;
4496 	nsd = &pf->stats;
4497 
4498 	/* Gather up the stats that the hw collects */
4499 	ixl_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
4500 			   vsi->stat_offsets_loaded,
4501 			   &oes->tx_errors, &es->tx_errors);
4502 	ixl_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
4503 			   vsi->stat_offsets_loaded,
4504 			   &oes->rx_discards, &es->rx_discards);
4505 
4506 	ixl_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
4507 			   I40E_GLV_GORCL(stat_idx),
4508 			   vsi->stat_offsets_loaded,
4509 			   &oes->rx_bytes, &es->rx_bytes);
4510 	ixl_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
4511 			   I40E_GLV_UPRCL(stat_idx),
4512 			   vsi->stat_offsets_loaded,
4513 			   &oes->rx_unicast, &es->rx_unicast);
4514 	ixl_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
4515 			   I40E_GLV_MPRCL(stat_idx),
4516 			   vsi->stat_offsets_loaded,
4517 			   &oes->rx_multicast, &es->rx_multicast);
4518 	ixl_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
4519 			   I40E_GLV_BPRCL(stat_idx),
4520 			   vsi->stat_offsets_loaded,
4521 			   &oes->rx_broadcast, &es->rx_broadcast);
4522 
4523 	ixl_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
4524 			   I40E_GLV_GOTCL(stat_idx),
4525 			   vsi->stat_offsets_loaded,
4526 			   &oes->tx_bytes, &es->tx_bytes);
4527 	ixl_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
4528 			   I40E_GLV_UPTCL(stat_idx),
4529 			   vsi->stat_offsets_loaded,
4530 			   &oes->tx_unicast, &es->tx_unicast);
4531 	ixl_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
4532 			   I40E_GLV_MPTCL(stat_idx),
4533 			   vsi->stat_offsets_loaded,
4534 			   &oes->tx_multicast, &es->tx_multicast);
4535 	ixl_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
4536 			   I40E_GLV_BPTCL(stat_idx),
4537 			   vsi->stat_offsets_loaded,
4538 			   &oes->tx_broadcast, &es->tx_broadcast);
4539 	vsi->stat_offsets_loaded = true;
4540 }
4541 
4542 static void
4543 ixl_update_vsi_stats(struct ixl_vsi *vsi)
4544 {
4545 	struct ixl_pf		*pf;
4546 	struct ifnet		*ifp;
4547 	struct i40e_eth_stats	*es;
4548 	u64			tx_discards;
4549 
4550 	struct i40e_hw_port_stats *nsd;
4551 
4552 	pf = vsi->back;
4553 	ifp = vsi->ifp;
4554 	es = &vsi->eth_stats;
4555 	nsd = &pf->stats;
4556 
4557 	ixl_update_eth_stats(vsi);
4558 
4559 	tx_discards = es->tx_discards + nsd->tx_dropped_link_down;
4560 	for (int i = 0; i < vsi->num_queues; i++)
4561 		tx_discards += vsi->queues[i].txr.br->br_drops;
4562 
4563 	/* Update ifnet stats */
4564 	IXL_SET_IPACKETS(vsi, es->rx_unicast +
4565 	                   es->rx_multicast +
4566 			   es->rx_broadcast);
4567 	IXL_SET_OPACKETS(vsi, es->tx_unicast +
4568 	                   es->tx_multicast +
4569 			   es->tx_broadcast);
4570 	IXL_SET_IBYTES(vsi, es->rx_bytes);
4571 	IXL_SET_OBYTES(vsi, es->tx_bytes);
4572 	IXL_SET_IMCASTS(vsi, es->rx_multicast);
4573 	IXL_SET_OMCASTS(vsi, es->tx_multicast);
4574 
4575 	IXL_SET_IERRORS(vsi, nsd->crc_errors + nsd->illegal_bytes +
4576 	    nsd->rx_undersize + nsd->rx_oversize + nsd->rx_fragments +
4577 	    nsd->rx_jabber);
4578 	IXL_SET_OERRORS(vsi, es->tx_errors);
4579 	IXL_SET_IQDROPS(vsi, es->rx_discards + nsd->eth.rx_discards);
4580 	IXL_SET_OQDROPS(vsi, tx_discards);
4581 	IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol);
4582 	IXL_SET_COLLISIONS(vsi, 0);
4583 }
4584 
4585 /**
4586  * Reset all of the stats for the given pf
4587  **/
4588 void ixl_pf_reset_stats(struct ixl_pf *pf)
4589 {
4590 	bzero(&pf->stats, sizeof(struct i40e_hw_port_stats));
4591 	bzero(&pf->stats_offsets, sizeof(struct i40e_hw_port_stats));
4592 	pf->stat_offsets_loaded = false;
4593 }
4594 
4595 /**
4596  * Resets all stats of the given vsi
4597  **/
4598 void ixl_vsi_reset_stats(struct ixl_vsi *vsi)
4599 {
4600 	bzero(&vsi->eth_stats, sizeof(struct i40e_eth_stats));
4601 	bzero(&vsi->eth_stats_offsets, sizeof(struct i40e_eth_stats));
4602 	vsi->stat_offsets_loaded = false;
4603 }
4604 
4605 /**
4606  * Read and update a 48 bit stat from the hw
4607  *
4608  * Since the device stats are not reset at PFReset, they likely will not
4609  * be zeroed when the driver starts.  We'll save the first values read
4610  * and use them as offsets to be subtracted from the raw values in order
4611  * to report stats that count from zero.
4612  **/
4613 static void
4614 ixl_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
4615 	bool offset_loaded, u64 *offset, u64 *stat)
4616 {
4617 	u64 new_data;
4618 
4619 #if defined(__FreeBSD__) && (__FreeBSD_version >= 1000000) && defined(__amd64__)
4620 	new_data = rd64(hw, loreg);
4621 #else
4622 	/*
4623 	 * Use two rd32's instead of one rd64; FreeBSD versions before
4624 	 * 10 don't support 8 byte bus reads/writes.
4625 	 */
4626 	new_data = rd32(hw, loreg);
4627 	new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
4628 #endif
4629 
4630 	if (!offset_loaded)
4631 		*offset = new_data;
4632 	if (new_data >= *offset)
4633 		*stat = new_data - *offset;
4634 	else
4635 		*stat = (new_data + ((u64)1 << 48)) - *offset;
4636 	*stat &= 0xFFFFFFFFFFFFULL;
4637 }
4638 
4639 /**
4640  * Read and update a 32 bit stat from the hw
4641  **/
4642 static void
4643 ixl_stat_update32(struct i40e_hw *hw, u32 reg,
4644 	bool offset_loaded, u64 *offset, u64 *stat)
4645 {
4646 	u32 new_data;
4647 
4648 	new_data = rd32(hw, reg);
4649 	if (!offset_loaded)
4650 		*offset = new_data;
4651 	if (new_data >= *offset)
4652 		*stat = (u32)(new_data - *offset);
4653 	else
4654 		*stat = (u32)((new_data + ((u64)1 << 32)) - *offset);
4655 }
4656 
4657 /*
4658 ** Set flow control using sysctl:
4659 ** 	0 - off
4660 **	1 - rx pause
4661 **	2 - tx pause
4662 **	3 - full
4663 */
4664 static int
4665 ixl_set_flowcntl(SYSCTL_HANDLER_ARGS)
4666 {
4667 	/*
4668 	 * TODO: ensure flow control is disabled if
4669 	 * priority flow control is enabled
4670 	 *
4671 	 * TODO: ensure tx CRC by hardware should be enabled
4672 	 * if tx flow control is enabled.
4673 	 */
4674 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
4675 	struct i40e_hw *hw = &pf->hw;
4676 	device_t dev = pf->dev;
4677 	int error = 0;
4678 	enum i40e_status_code aq_error = 0;
4679 	u8 fc_aq_err = 0;
4680 
4681 	/* Get request */
4682 	error = sysctl_handle_int(oidp, &pf->fc, 0, req);
4683 	if ((error) || (req->newptr == NULL))
4684 		return (error);
4685 	if (pf->fc < 0 || pf->fc > 3) {
4686 		device_printf(dev,
4687 		    "Invalid fc mode; valid modes are 0 through 3\n");
4688 		return (EINVAL);
4689 	}
4690 
4691 	/*
4692 	** Changing flow control mode currently does not work on
4693 	** 40GBASE-CR4 PHYs
4694 	*/
4695 	if (hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4
4696 	    || hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4_CU) {
4697 		device_printf(dev, "Changing flow control mode unsupported"
4698 		    " on 40GBase-CR4 media.\n");
4699 		return (ENODEV);
4700 	}
4701 
4702 	/* Set fc ability for port */
4703 	hw->fc.requested_mode = pf->fc;
4704 	aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE);
4705 	if (aq_error) {
4706 		device_printf(dev,
4707 		    "%s: Error setting new fc mode %d; fc_err %#x\n",
4708 		    __func__, aq_error, fc_aq_err);
4709 		return (EAGAIN);
4710 	}
4711 
4712 	return (0);
4713 }
4714 
4715 static int
4716 ixl_current_speed(SYSCTL_HANDLER_ARGS)
4717 {
4718 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
4719 	struct i40e_hw *hw = &pf->hw;
4720 	int error = 0, index = 0;
4721 
4722 	char *speeds[] = {
4723 		"Unknown",
4724 		"100M",
4725 		"1G",
4726 		"10G",
4727 		"40G",
4728 		"20G"
4729 	};
4730 
4731 	ixl_update_link_status(pf);
4732 
4733 	switch (hw->phy.link_info.link_speed) {
4734 	case I40E_LINK_SPEED_100MB:
4735 		index = 1;
4736 		break;
4737 	case I40E_LINK_SPEED_1GB:
4738 		index = 2;
4739 		break;
4740 	case I40E_LINK_SPEED_10GB:
4741 		index = 3;
4742 		break;
4743 	case I40E_LINK_SPEED_40GB:
4744 		index = 4;
4745 		break;
4746 	case I40E_LINK_SPEED_20GB:
4747 		index = 5;
4748 		break;
4749 	case I40E_LINK_SPEED_UNKNOWN:
4750 	default:
4751 		index = 0;
4752 		break;
4753 	}
4754 
4755 	error = sysctl_handle_string(oidp, speeds[index],
4756 	    strlen(speeds[index]), req);
4757 	return (error);
4758 }
4759 
4760 static int
4761 ixl_set_advertised_speeds(struct ixl_pf *pf, int speeds)
4762 {
4763 	struct i40e_hw *hw = &pf->hw;
4764 	device_t dev = pf->dev;
4765 	struct i40e_aq_get_phy_abilities_resp abilities;
4766 	struct i40e_aq_set_phy_config config;
4767 	enum i40e_status_code aq_error = 0;
4768 
4769 	/* Get current capability information */
4770 	aq_error = i40e_aq_get_phy_capabilities(hw,
4771 	    FALSE, FALSE, &abilities, NULL);
4772 	if (aq_error) {
4773 		device_printf(dev,
4774 		    "%s: Error getting phy capabilities %d,"
4775 		    " aq error: %d\n", __func__, aq_error,
4776 		    hw->aq.asq_last_status);
4777 		return (EAGAIN);
4778 	}
4779 
4780 	/* Prepare new config */
4781 	bzero(&config, sizeof(config));
4782 	config.phy_type = abilities.phy_type;
4783 	config.abilities = abilities.abilities
4784 	    | I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
4785 	config.eee_capability = abilities.eee_capability;
4786 	config.eeer = abilities.eeer_val;
4787 	config.low_power_ctrl = abilities.d3_lpan;
4788 	/* Translate into aq cmd link_speed */
4789 	if (speeds & 0x8)
4790 		config.link_speed |= I40E_LINK_SPEED_20GB;
4791 	if (speeds & 0x4)
4792 		config.link_speed |= I40E_LINK_SPEED_10GB;
4793 	if (speeds & 0x2)
4794 		config.link_speed |= I40E_LINK_SPEED_1GB;
4795 	if (speeds & 0x1)
4796 		config.link_speed |= I40E_LINK_SPEED_100MB;
4797 
4798 	/* Do aq command & restart link */
4799 	aq_error = i40e_aq_set_phy_config(hw, &config, NULL);
4800 	if (aq_error) {
4801 		device_printf(dev,
4802 		    "%s: Error setting new phy config %d,"
4803 		    " aq error: %d\n", __func__, aq_error,
4804 		    hw->aq.asq_last_status);
4805 		return (EAGAIN);
4806 	}
4807 
4808 	/*
4809 	** This seems a bit heavy handed, but we
4810 	** need to get a reinit on some devices
4811 	*/
4812 	IXL_PF_LOCK(pf);
4813 	ixl_stop(pf);
4814 	ixl_init_locked(pf);
4815 	IXL_PF_UNLOCK(pf);
4816 
4817 	return (0);
4818 }
4819 
4820 /*
4821 ** Control link advertise speed:
4822 **	Flags:
4823 **	0x1 - advertise 100 Mb
4824 **	0x2 - advertise 1G
4825 **	0x4 - advertise 10G
4826 **	0x8 - advertise 20G
4827 **
4828 ** Does not work on 40G devices.
4829 */
4830 static int
4831 ixl_set_advertise(SYSCTL_HANDLER_ARGS)
4832 {
4833 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
4834 	struct i40e_hw *hw = &pf->hw;
4835 	device_t dev = pf->dev;
4836 	int requested_ls = 0;
4837 	int error = 0;
4838 
4839 	/*
4840 	** FW doesn't support changing advertised speed
4841 	** for 40G devices; speed is always 40G.
4842 	*/
4843 	if (i40e_is_40G_device(hw->device_id))
4844 		return (ENODEV);
4845 
4846 	/* Read in new mode */
4847 	requested_ls = pf->advertised_speed;
4848 	error = sysctl_handle_int(oidp, &requested_ls, 0, req);
4849 	if ((error) || (req->newptr == NULL))
4850 		return (error);
4851 	/* Check for sane value */
4852 	if (requested_ls < 0x1 || requested_ls > 0xE) {
4853 		device_printf(dev, "Invalid advertised speed; "
4854 		    "valid modes are 0x1 through 0xE\n");
4855 		return (EINVAL);
4856 	}
4857 	/* Then check for validity based on adapter type */
4858 	switch (hw->device_id) {
4859 	case I40E_DEV_ID_10G_BASE_T:
4860 		if (requested_ls & 0x8) {
4861 			device_printf(dev,
4862 			    "20Gbs speed not supported on this device.\n");
4863 			return (EINVAL);
4864 		}
4865 		break;
4866 	case I40E_DEV_ID_20G_KR2:
4867 		if (requested_ls & 0x1) {
4868 			device_printf(dev,
4869 			    "100Mbs speed not supported on this device.\n");
4870 			return (EINVAL);
4871 		}
4872 		break;
4873 	default:
4874 		if (requested_ls & ~0x6) {
4875 			device_printf(dev,
4876 			    "Only 1/10Gbs speeds are supported on this device.\n");
4877 			return (EINVAL);
4878 		}
4879 		break;
4880 	}
4881 
4882 	/* Exit if no change */
4883 	if (pf->advertised_speed == requested_ls)
4884 		return (0);
4885 
4886 	error = ixl_set_advertised_speeds(pf, requested_ls);
4887 	if (error)
4888 		return (error);
4889 
4890 	pf->advertised_speed = requested_ls;
4891 	ixl_update_link_status(pf);
4892 	return (0);
4893 }
4894 
4895 /*
4896 ** Get the width and transaction speed of
4897 ** the bus this adapter is plugged into.
4898 */
4899 static u16
4900 ixl_get_bus_info(struct i40e_hw *hw, device_t dev)
4901 {
4902         u16                     link;
4903         u32                     offset;
4904 
4905 
4906         /* Get the PCI Express Capabilities offset */
4907         pci_find_cap(dev, PCIY_EXPRESS, &offset);
4908 
4909         /* ...and read the Link Status Register */
4910         link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
4911 
4912         switch (link & I40E_PCI_LINK_WIDTH) {
4913         case I40E_PCI_LINK_WIDTH_1:
4914                 hw->bus.width = i40e_bus_width_pcie_x1;
4915                 break;
4916         case I40E_PCI_LINK_WIDTH_2:
4917                 hw->bus.width = i40e_bus_width_pcie_x2;
4918                 break;
4919         case I40E_PCI_LINK_WIDTH_4:
4920                 hw->bus.width = i40e_bus_width_pcie_x4;
4921                 break;
4922         case I40E_PCI_LINK_WIDTH_8:
4923                 hw->bus.width = i40e_bus_width_pcie_x8;
4924                 break;
4925         default:
4926                 hw->bus.width = i40e_bus_width_unknown;
4927                 break;
4928         }
4929 
4930         switch (link & I40E_PCI_LINK_SPEED) {
4931         case I40E_PCI_LINK_SPEED_2500:
4932                 hw->bus.speed = i40e_bus_speed_2500;
4933                 break;
4934         case I40E_PCI_LINK_SPEED_5000:
4935                 hw->bus.speed = i40e_bus_speed_5000;
4936                 break;
4937         case I40E_PCI_LINK_SPEED_8000:
4938                 hw->bus.speed = i40e_bus_speed_8000;
4939                 break;
4940         default:
4941                 hw->bus.speed = i40e_bus_speed_unknown;
4942                 break;
4943         }
4944 
4945 
4946         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
4947             ((hw->bus.speed == i40e_bus_speed_8000) ? "8.0GT/s":
4948             (hw->bus.speed == i40e_bus_speed_5000) ? "5.0GT/s":
4949             (hw->bus.speed == i40e_bus_speed_2500) ? "2.5GT/s":"Unknown"),
4950             (hw->bus.width == i40e_bus_width_pcie_x8) ? "Width x8" :
4951             (hw->bus.width == i40e_bus_width_pcie_x4) ? "Width x4" :
4952             (hw->bus.width == i40e_bus_width_pcie_x1) ? "Width x1" :
4953             ("Unknown"));
4954 
4955         if ((hw->bus.width <= i40e_bus_width_pcie_x8) &&
4956             (hw->bus.speed < i40e_bus_speed_8000)) {
4957                 device_printf(dev, "PCI-Express bandwidth available"
4958                     " for this device\n     may be insufficient for"
4959                     " optimal performance.\n");
4960                 device_printf(dev, "For expected performance a x8 "
4961                     "PCIE Gen3 slot is required.\n");
4962         }
4963 
4964         return (link);
4965 }
4966 
4967 static int
4968 ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS)
4969 {
4970 	struct ixl_pf	*pf = (struct ixl_pf *)arg1;
4971 	struct i40e_hw	*hw = &pf->hw;
4972 	char		buf[32];
4973 
4974 	snprintf(buf, sizeof(buf),
4975 	    "f%d.%d a%d.%d n%02x.%02x e%08x",
4976 	    hw->aq.fw_maj_ver, hw->aq.fw_min_ver,
4977 	    hw->aq.api_maj_ver, hw->aq.api_min_ver,
4978 	    (hw->nvm.version & IXL_NVM_VERSION_HI_MASK) >>
4979 	    IXL_NVM_VERSION_HI_SHIFT,
4980 	    (hw->nvm.version & IXL_NVM_VERSION_LO_MASK) >>
4981 	    IXL_NVM_VERSION_LO_SHIFT,
4982 	    hw->nvm.eetrack);
4983 	return (sysctl_handle_string(oidp, buf, strlen(buf), req));
4984 }
4985 
4986 
4987 #ifdef IXL_DEBUG_SYSCTL
4988 static int
4989 ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS)
4990 {
4991 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
4992 	struct i40e_hw *hw = &pf->hw;
4993 	struct i40e_link_status link_status;
4994 	char buf[512];
4995 
4996 	enum i40e_status_code aq_error = 0;
4997 
4998 	aq_error = i40e_aq_get_link_info(hw, TRUE, &link_status, NULL);
4999 	if (aq_error) {
5000 		printf("i40e_aq_get_link_info() error %d\n", aq_error);
5001 		return (EPERM);
5002 	}
5003 
5004 	sprintf(buf, "\n"
5005 	    "PHY Type : %#04x\n"
5006 	    "Speed    : %#04x\n"
5007 	    "Link info: %#04x\n"
5008 	    "AN info  : %#04x\n"
5009 	    "Ext info : %#04x",
5010 	    link_status.phy_type, link_status.link_speed,
5011 	    link_status.link_info, link_status.an_info,
5012 	    link_status.ext_info);
5013 
5014 	return (sysctl_handle_string(oidp, buf, strlen(buf), req));
5015 }
5016 
5017 static int
5018 ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS)
5019 {
5020 	struct ixl_pf		*pf = (struct ixl_pf *)arg1;
5021 	struct i40e_hw		*hw = &pf->hw;
5022 	char			buf[512];
5023 	enum i40e_status_code	aq_error = 0;
5024 
5025 	struct i40e_aq_get_phy_abilities_resp abilities;
5026 
5027 	aq_error = i40e_aq_get_phy_capabilities(hw,
5028 	    TRUE, FALSE, &abilities, NULL);
5029 	if (aq_error) {
5030 		printf("i40e_aq_get_phy_capabilities() error %d\n", aq_error);
5031 		return (EPERM);
5032 	}
5033 
5034 	sprintf(buf, "\n"
5035 	    "PHY Type : %#010x\n"
5036 	    "Speed    : %#04x\n"
5037 	    "Abilities: %#04x\n"
5038 	    "EEE cap  : %#06x\n"
5039 	    "EEER reg : %#010x\n"
5040 	    "D3 Lpan  : %#04x",
5041 	    abilities.phy_type, abilities.link_speed,
5042 	    abilities.abilities, abilities.eee_capability,
5043 	    abilities.eeer_val, abilities.d3_lpan);
5044 
5045 	return (sysctl_handle_string(oidp, buf, strlen(buf), req));
5046 }
5047 
5048 static int
5049 ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS)
5050 {
5051 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
5052 	struct ixl_vsi *vsi = &pf->vsi;
5053 	struct ixl_mac_filter *f;
5054 	char *buf, *buf_i;
5055 
5056 	int error = 0;
5057 	int ftl_len = 0;
5058 	int ftl_counter = 0;
5059 	int buf_len = 0;
5060 	int entry_len = 42;
5061 
5062 	SLIST_FOREACH(f, &vsi->ftl, next) {
5063 		ftl_len++;
5064 	}
5065 
5066 	if (ftl_len < 1) {
5067 		sysctl_handle_string(oidp, "(none)", 6, req);
5068 		return (0);
5069 	}
5070 
5071 	buf_len = sizeof(char) * (entry_len + 1) * ftl_len + 2;
5072 	buf = buf_i = malloc(buf_len, M_DEVBUF, M_NOWAIT);
5073 
5074 	sprintf(buf_i++, "\n");
5075 	SLIST_FOREACH(f, &vsi->ftl, next) {
5076 		sprintf(buf_i,
5077 		    MAC_FORMAT ", vlan %4d, flags %#06x",
5078 		    MAC_FORMAT_ARGS(f->macaddr), f->vlan, f->flags);
5079 		buf_i += entry_len;
5080 		/* don't print '\n' for last entry */
5081 		if (++ftl_counter != ftl_len) {
5082 			sprintf(buf_i, "\n");
5083 			buf_i++;
5084 		}
5085 	}
5086 
5087 	error = sysctl_handle_string(oidp, buf, strlen(buf), req);
5088 	if (error)
5089 		printf("sysctl error: %d\n", error);
5090 	free(buf, M_DEVBUF);
5091 	return error;
5092 }
5093 
5094 #define IXL_SW_RES_SIZE 0x14
5095 static int
5096 ixl_res_alloc_cmp(const void *a, const void *b)
5097 {
5098 	const struct i40e_aqc_switch_resource_alloc_element_resp *one, *two;
5099 	one = (const struct i40e_aqc_switch_resource_alloc_element_resp *)a;
5100 	two = (const struct i40e_aqc_switch_resource_alloc_element_resp *)b;
5101 
5102 	return ((int)one->resource_type - (int)two->resource_type);
5103 }
5104 
5105 static int
5106 ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS)
5107 {
5108 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
5109 	struct i40e_hw *hw = &pf->hw;
5110 	device_t dev = pf->dev;
5111 	struct sbuf *buf;
5112 	int error = 0;
5113 
5114 	u8 num_entries;
5115 	struct i40e_aqc_switch_resource_alloc_element_resp resp[IXL_SW_RES_SIZE];
5116 
5117 	buf = sbuf_new_for_sysctl(NULL, NULL, 0, req);
5118 	if (!buf) {
5119 		device_printf(dev, "Could not allocate sbuf for output.\n");
5120 		return (ENOMEM);
5121 	}
5122 
5123 	bzero(resp, sizeof(resp));
5124 	error = i40e_aq_get_switch_resource_alloc(hw, &num_entries,
5125 				resp,
5126 				IXL_SW_RES_SIZE,
5127 				NULL);
5128 	if (error) {
5129 		device_printf(dev,
5130 		    "%s: get_switch_resource_alloc() error %d, aq error %d\n",
5131 		    __func__, error, hw->aq.asq_last_status);
5132 		sbuf_delete(buf);
5133 		return error;
5134 	}
5135 
5136 	/* Sort entries by type for display */
5137 	qsort(resp, num_entries,
5138 	    sizeof(struct i40e_aqc_switch_resource_alloc_element_resp),
5139 	    &ixl_res_alloc_cmp);
5140 
5141 	sbuf_cat(buf, "\n");
5142 	sbuf_printf(buf, "# of entries: %d\n", num_entries);
5143 	sbuf_printf(buf,
5144 	    "Type | Guaranteed | Total | Used   | Un-allocated\n"
5145 	    "     | (this)     | (all) | (this) | (all)       \n");
5146 	for (int i = 0; i < num_entries; i++) {
5147 		sbuf_printf(buf,
5148 		    "%#4x | %10d   %5d   %6d   %12d",
5149 		    resp[i].resource_type,
5150 		    resp[i].guaranteed,
5151 		    resp[i].total,
5152 		    resp[i].used,
5153 		    resp[i].total_unalloced);
5154 		if (i < num_entries - 1)
5155 			sbuf_cat(buf, "\n");
5156 	}
5157 
5158 	error = sbuf_finish(buf);
5159 	sbuf_delete(buf);
5160 
5161 	return (error);
5162 }
5163 
5164 /*
5165 ** Caller must init and delete sbuf; this function will clear and
5166 ** finish it for caller.
5167 */
5168 static char *
5169 ixl_switch_element_string(struct sbuf *s, u16 seid, bool uplink)
5170 {
5171 	sbuf_clear(s);
5172 
5173 	if (seid == 0 && uplink)
5174 		sbuf_cat(s, "Network");
5175 	else if (seid == 0)
5176 		sbuf_cat(s, "Host");
5177 	else if (seid == 1)
5178 		sbuf_cat(s, "EMP");
5179 	else if (seid <= 5)
5180 		sbuf_printf(s, "MAC %d", seid - 2);
5181 	else if (seid <= 15)
5182 		sbuf_cat(s, "Reserved");
5183 	else if (seid <= 31)
5184 		sbuf_printf(s, "PF %d", seid - 16);
5185 	else if (seid <= 159)
5186 		sbuf_printf(s, "VF %d", seid - 32);
5187 	else if (seid <= 287)
5188 		sbuf_cat(s, "Reserved");
5189 	else if (seid <= 511)
5190 		sbuf_cat(s, "Other"); // for other structures
5191 	else if (seid <= 895)
5192 		sbuf_printf(s, "VSI %d", seid - 512);
5193 	else if (seid <= 1023)
5194 		sbuf_printf(s, "Reserved");
5195 	else
5196 		sbuf_cat(s, "Invalid");
5197 
5198 	sbuf_finish(s);
5199 	return sbuf_data(s);
5200 }
5201 
5202 static int
5203 ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS)
5204 {
5205 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
5206 	struct i40e_hw *hw = &pf->hw;
5207 	device_t dev = pf->dev;
5208 	struct sbuf *buf;
5209 	struct sbuf *nmbuf;
5210 	int error = 0;
5211 	u8 aq_buf[I40E_AQ_LARGE_BUF];
5212 
5213 	u16 next = 0;
5214 	struct i40e_aqc_get_switch_config_resp *sw_config;
5215 	sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
5216 
5217 	buf = sbuf_new_for_sysctl(NULL, NULL, 0, req);
5218 	if (!buf) {
5219 		device_printf(dev, "Could not allocate sbuf for sysctl output.\n");
5220 		return (ENOMEM);
5221 	}
5222 
5223 	error = i40e_aq_get_switch_config(hw, sw_config,
5224 	    sizeof(aq_buf), &next, NULL);
5225 	if (error) {
5226 		device_printf(dev,
5227 		    "%s: aq_get_switch_config() error %d, aq error %d\n",
5228 		    __func__, error, hw->aq.asq_last_status);
5229 		sbuf_delete(buf);
5230 		return error;
5231 	}
5232 
5233 	nmbuf = sbuf_new_auto();
5234 	if (!nmbuf) {
5235 		device_printf(dev, "Could not allocate sbuf for name output.\n");
5236 		return (ENOMEM);
5237 	}
5238 
5239 	sbuf_cat(buf, "\n");
5240 	// Assuming <= 255 elements in switch
5241 	sbuf_printf(buf, "# of elements: %d\n", sw_config->header.num_reported);
5242 	/* Exclude:
5243 	** Revision -- all elements are revision 1 for now
5244 	*/
5245 	sbuf_printf(buf,
5246 	    "SEID (  Name  ) |  Uplink  | Downlink | Conn Type\n"
5247 	    "                |          |          | (uplink)\n");
5248 	for (int i = 0; i < sw_config->header.num_reported; i++) {
5249 		// "%4d (%8s) | %8s   %8s   %#8x",
5250 		sbuf_printf(buf, "%4d", sw_config->element[i].seid);
5251 		sbuf_cat(buf, " ");
5252 		sbuf_printf(buf, "(%8s)", ixl_switch_element_string(nmbuf,
5253 		    sw_config->element[i].seid, false));
5254 		sbuf_cat(buf, " | ");
5255 		sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf,
5256 		    sw_config->element[i].uplink_seid, true));
5257 		sbuf_cat(buf, "   ");
5258 		sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf,
5259 		    sw_config->element[i].downlink_seid, false));
5260 		sbuf_cat(buf, "   ");
5261 		sbuf_printf(buf, "%#8x", sw_config->element[i].connection_type);
5262 		if (i < sw_config->header.num_reported - 1)
5263 			sbuf_cat(buf, "\n");
5264 	}
5265 	sbuf_delete(nmbuf);
5266 
5267 	error = sbuf_finish(buf);
5268 	sbuf_delete(buf);
5269 
5270 	return (error);
5271 }
5272 #endif /* IXL_DEBUG_SYSCTL */
5273 
5274 
5275 #ifdef PCI_IOV
5276 static int
5277 ixl_vf_alloc_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
5278 {
5279 	struct i40e_hw *hw;
5280 	struct ixl_vsi *vsi;
5281 	struct i40e_vsi_context vsi_ctx;
5282 	int i;
5283 	uint16_t first_queue;
5284 	enum i40e_status_code code;
5285 
5286 	hw = &pf->hw;
5287 	vsi = &pf->vsi;
5288 
5289 	vsi_ctx.pf_num = hw->pf_id;
5290 	vsi_ctx.uplink_seid = pf->veb_seid;
5291 	vsi_ctx.connection_type = IXL_VSI_DATA_PORT;
5292 	vsi_ctx.vf_num = hw->func_caps.vf_base_id + vf->vf_num;
5293 	vsi_ctx.flags = I40E_AQ_VSI_TYPE_VF;
5294 
5295 	bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
5296 
5297 	vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_SWITCH_VALID);
5298 	vsi_ctx.info.switch_id = htole16(0);
5299 
5300 	vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_SECURITY_VALID);
5301 	vsi_ctx.info.sec_flags = 0;
5302 	if (vf->vf_flags & VF_FLAG_MAC_ANTI_SPOOF)
5303 		vsi_ctx.info.sec_flags |= I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK;
5304 
5305 	vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
5306 	vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
5307 	    I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
5308 
5309 	vsi_ctx.info.valid_sections |=
5310 	    htole16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
5311 	vsi_ctx.info.mapping_flags = htole16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
5312 	first_queue = vsi->num_queues + vf->vf_num * IXLV_MAX_QUEUES;
5313 	for (i = 0; i < IXLV_MAX_QUEUES; i++)
5314 		vsi_ctx.info.queue_mapping[i] = htole16(first_queue + i);
5315 	for (; i < nitems(vsi_ctx.info.queue_mapping); i++)
5316 		vsi_ctx.info.queue_mapping[i] = htole16(I40E_AQ_VSI_QUEUE_MASK);
5317 
5318 	vsi_ctx.info.tc_mapping[0] = htole16(
5319 	    (0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
5320 	    (1 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
5321 
5322 	code = i40e_aq_add_vsi(hw, &vsi_ctx, NULL);
5323 	if (code != I40E_SUCCESS)
5324 		return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5325 	vf->vsi.seid = vsi_ctx.seid;
5326 	vf->vsi.vsi_num = vsi_ctx.vsi_number;
5327 	vf->vsi.first_queue = first_queue;
5328 	vf->vsi.num_queues = IXLV_MAX_QUEUES;
5329 
5330 	code = i40e_aq_get_vsi_params(hw, &vsi_ctx, NULL);
5331 	if (code != I40E_SUCCESS)
5332 		return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5333 
5334 	code = i40e_aq_config_vsi_bw_limit(hw, vf->vsi.seid, 0, 0, NULL);
5335 	if (code != I40E_SUCCESS) {
5336 		device_printf(pf->dev, "Failed to disable BW limit: %d\n",
5337 		    ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5338 		return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5339 	}
5340 
5341 	memcpy(&vf->vsi.info, &vsi_ctx.info, sizeof(vf->vsi.info));
5342 	return (0);
5343 }
5344 
5345 static int
5346 ixl_vf_setup_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
5347 {
5348 	struct i40e_hw *hw;
5349 	int error;
5350 
5351 	hw = &pf->hw;
5352 
5353 	error = ixl_vf_alloc_vsi(pf, vf);
5354 	if (error != 0)
5355 		return (error);
5356 
5357 	vf->vsi.hw_filters_add = 0;
5358 	vf->vsi.hw_filters_del = 0;
5359 	ixl_add_filter(&vf->vsi, ixl_bcast_addr, IXL_VLAN_ANY);
5360 	ixl_reconfigure_filters(&vf->vsi);
5361 
5362 	return (0);
5363 }
5364 
5365 static void
5366 ixl_vf_map_vsi_queue(struct i40e_hw *hw, struct ixl_vf *vf, int qnum,
5367     uint32_t val)
5368 {
5369 	uint32_t qtable;
5370 	int index, shift;
5371 
5372 	/*
5373 	 * Two queues are mapped in a single register, so we have to do some
5374 	 * gymnastics to convert the queue number into a register index and
5375 	 * shift.
5376 	 */
5377 	index = qnum / 2;
5378 	shift = (qnum % 2) * I40E_VSILAN_QTABLE_QINDEX_1_SHIFT;
5379 
5380 	qtable = rd32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num));
5381 	qtable &= ~(I40E_VSILAN_QTABLE_QINDEX_0_MASK << shift);
5382 	qtable |= val << shift;
5383 	wr32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num), qtable);
5384 }
5385 
5386 static void
5387 ixl_vf_map_queues(struct ixl_pf *pf, struct ixl_vf *vf)
5388 {
5389 	struct i40e_hw *hw;
5390 	uint32_t qtable;
5391 	int i;
5392 
5393 	hw = &pf->hw;
5394 
5395 	/*
5396 	 * Contiguous mappings aren't actually supported by the hardware,
5397 	 * so we have to use non-contiguous mappings.
5398 	 */
5399 	wr32(hw, I40E_VSILAN_QBASE(vf->vsi.vsi_num),
5400 	     I40E_VSILAN_QBASE_VSIQTABLE_ENA_MASK);
5401 
5402 	wr32(hw, I40E_VPLAN_MAPENA(vf->vf_num),
5403 	    I40E_VPLAN_MAPENA_TXRX_ENA_MASK);
5404 
5405 	for (i = 0; i < vf->vsi.num_queues; i++) {
5406 		qtable = (vf->vsi.first_queue + i) <<
5407 		    I40E_VPLAN_QTABLE_QINDEX_SHIFT;
5408 
5409 		wr32(hw, I40E_VPLAN_QTABLE(i, vf->vf_num), qtable);
5410 	}
5411 
5412 	/* Map queues allocated to VF to its VSI. */
5413 	for (i = 0; i < vf->vsi.num_queues; i++)
5414 		ixl_vf_map_vsi_queue(hw, vf, i, vf->vsi.first_queue + i);
5415 
5416 	/* Set rest of VSI queues as unused. */
5417 	for (; i < IXL_MAX_VSI_QUEUES; i++)
5418 		ixl_vf_map_vsi_queue(hw, vf, i,
5419 		    I40E_VSILAN_QTABLE_QINDEX_0_MASK);
5420 
5421 	ixl_flush(hw);
5422 }
5423 
5424 static void
5425 ixl_vf_vsi_release(struct ixl_pf *pf, struct ixl_vsi *vsi)
5426 {
5427 	struct i40e_hw *hw;
5428 
5429 	hw = &pf->hw;
5430 
5431 	if (vsi->seid == 0)
5432 		return;
5433 
5434 	i40e_aq_delete_element(hw, vsi->seid, NULL);
5435 }
5436 
5437 static void
5438 ixl_vf_disable_queue_intr(struct i40e_hw *hw, uint32_t vfint_reg)
5439 {
5440 
5441 	wr32(hw, vfint_reg, I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
5442 	ixl_flush(hw);
5443 }
5444 
5445 static void
5446 ixl_vf_unregister_intr(struct i40e_hw *hw, uint32_t vpint_reg)
5447 {
5448 
5449 	wr32(hw, vpint_reg, I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_MASK |
5450 	    I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK);
5451 	ixl_flush(hw);
5452 }
5453 
5454 static void
5455 ixl_vf_release_resources(struct ixl_pf *pf, struct ixl_vf *vf)
5456 {
5457 	struct i40e_hw *hw;
5458 	uint32_t vfint_reg, vpint_reg;
5459 	int i;
5460 
5461 	hw = &pf->hw;
5462 
5463 	ixl_vf_vsi_release(pf, &vf->vsi);
5464 
5465 	/* Index 0 has a special register. */
5466 	ixl_vf_disable_queue_intr(hw, I40E_VFINT_DYN_CTL0(vf->vf_num));
5467 
5468 	for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
5469 		vfint_reg = IXL_VFINT_DYN_CTLN_REG(hw, i , vf->vf_num);
5470 		ixl_vf_disable_queue_intr(hw, vfint_reg);
5471 	}
5472 
5473 	/* Index 0 has a special register. */
5474 	ixl_vf_unregister_intr(hw, I40E_VPINT_LNKLST0(vf->vf_num));
5475 
5476 	for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
5477 		vpint_reg = IXL_VPINT_LNKLSTN_REG(hw, i, vf->vf_num);
5478 		ixl_vf_unregister_intr(hw, vpint_reg);
5479 	}
5480 
5481 	vf->vsi.num_queues = 0;
5482 }
5483 
5484 static int
5485 ixl_flush_pcie(struct ixl_pf *pf, struct ixl_vf *vf)
5486 {
5487 	struct i40e_hw *hw;
5488 	int i;
5489 	uint16_t global_vf_num;
5490 	uint32_t ciad;
5491 
5492 	hw = &pf->hw;
5493 	global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
5494 
5495 	wr32(hw, I40E_PF_PCI_CIAA, IXL_PF_PCI_CIAA_VF_DEVICE_STATUS |
5496 	     (global_vf_num << I40E_PF_PCI_CIAA_VF_NUM_SHIFT));
5497 	for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
5498 		ciad = rd32(hw, I40E_PF_PCI_CIAD);
5499 		if ((ciad & IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK) == 0)
5500 			return (0);
5501 		DELAY(1);
5502 	}
5503 
5504 	return (ETIMEDOUT);
5505 }
5506 
5507 static void
5508 ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf)
5509 {
5510 	struct i40e_hw *hw;
5511 	uint32_t vfrtrig;
5512 
5513 	hw = &pf->hw;
5514 
5515 	vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
5516 	vfrtrig |= I40E_VPGEN_VFRTRIG_VFSWR_MASK;
5517 	wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
5518 	ixl_flush(hw);
5519 
5520 	ixl_reinit_vf(pf, vf);
5521 }
5522 
5523 static void
5524 ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf)
5525 {
5526 	struct i40e_hw *hw;
5527 	uint32_t vfrstat, vfrtrig;
5528 	int i, error;
5529 
5530 	hw = &pf->hw;
5531 
5532 	error = ixl_flush_pcie(pf, vf);
5533 	if (error != 0)
5534 		device_printf(pf->dev,
5535 		    "Timed out waiting for PCIe activity to stop on VF-%d\n",
5536 		    vf->vf_num);
5537 
5538 	for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
5539 		DELAY(10);
5540 
5541 		vfrstat = rd32(hw, I40E_VPGEN_VFRSTAT(vf->vf_num));
5542 		if (vfrstat & I40E_VPGEN_VFRSTAT_VFRD_MASK)
5543 			break;
5544 	}
5545 
5546 	if (i == IXL_VF_RESET_TIMEOUT)
5547 		device_printf(pf->dev, "VF %d failed to reset\n", vf->vf_num);
5548 
5549 	wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_COMPLETED);
5550 
5551 	vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
5552 	vfrtrig &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK;
5553 	wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
5554 
5555 	if (vf->vsi.seid != 0)
5556 		ixl_disable_rings(&vf->vsi);
5557 
5558 	ixl_vf_release_resources(pf, vf);
5559 	ixl_vf_setup_vsi(pf, vf);
5560 	ixl_vf_map_queues(pf, vf);
5561 
5562 	wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_VFACTIVE);
5563 	ixl_flush(hw);
5564 }
5565 
5566 static const char *
5567 ixl_vc_opcode_str(uint16_t op)
5568 {
5569 
5570 	switch (op) {
5571 	case I40E_VIRTCHNL_OP_VERSION:
5572 		return ("VERSION");
5573 	case I40E_VIRTCHNL_OP_RESET_VF:
5574 		return ("RESET_VF");
5575 	case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
5576 		return ("GET_VF_RESOURCES");
5577 	case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
5578 		return ("CONFIG_TX_QUEUE");
5579 	case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
5580 		return ("CONFIG_RX_QUEUE");
5581 	case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
5582 		return ("CONFIG_VSI_QUEUES");
5583 	case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
5584 		return ("CONFIG_IRQ_MAP");
5585 	case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
5586 		return ("ENABLE_QUEUES");
5587 	case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
5588 		return ("DISABLE_QUEUES");
5589 	case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
5590 		return ("ADD_ETHER_ADDRESS");
5591 	case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
5592 		return ("DEL_ETHER_ADDRESS");
5593 	case I40E_VIRTCHNL_OP_ADD_VLAN:
5594 		return ("ADD_VLAN");
5595 	case I40E_VIRTCHNL_OP_DEL_VLAN:
5596 		return ("DEL_VLAN");
5597 	case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
5598 		return ("CONFIG_PROMISCUOUS_MODE");
5599 	case I40E_VIRTCHNL_OP_GET_STATS:
5600 		return ("GET_STATS");
5601 	case I40E_VIRTCHNL_OP_FCOE:
5602 		return ("FCOE");
5603 	case I40E_VIRTCHNL_OP_EVENT:
5604 		return ("EVENT");
5605 	default:
5606 		return ("UNKNOWN");
5607 	}
5608 }
5609 
5610 static int
5611 ixl_vc_opcode_level(uint16_t opcode)
5612 {
5613 
5614 	switch (opcode) {
5615 	case I40E_VIRTCHNL_OP_GET_STATS:
5616 		return (10);
5617 	default:
5618 		return (5);
5619 	}
5620 }
5621 
5622 static void
5623 ixl_send_vf_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
5624     enum i40e_status_code status, void *msg, uint16_t len)
5625 {
5626 	struct i40e_hw *hw;
5627 	int global_vf_id;
5628 
5629 	hw = &pf->hw;
5630 	global_vf_id = hw->func_caps.vf_base_id + vf->vf_num;
5631 
5632 	I40E_VC_DEBUG(pf, ixl_vc_opcode_level(op),
5633 	    "Sending msg (op=%s[%d], status=%d) to VF-%d\n",
5634 	    ixl_vc_opcode_str(op), op, status, vf->vf_num);
5635 
5636 	i40e_aq_send_msg_to_vf(hw, global_vf_id, op, status, msg, len, NULL);
5637 }
5638 
5639 static void
5640 ixl_send_vf_ack(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op)
5641 {
5642 
5643 	ixl_send_vf_msg(pf, vf, op, I40E_SUCCESS, NULL, 0);
5644 }
5645 
5646 static void
5647 ixl_send_vf_nack_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
5648     enum i40e_status_code status, const char *file, int line)
5649 {
5650 
5651 	I40E_VC_DEBUG(pf, 1,
5652 	    "Sending NACK (op=%s[%d], err=%d) to VF-%d from %s:%d\n",
5653 	    ixl_vc_opcode_str(op), op, status, vf->vf_num, file, line);
5654 	ixl_send_vf_msg(pf, vf, op, status, NULL, 0);
5655 }
5656 
5657 static void
5658 ixl_vf_version_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5659     uint16_t msg_size)
5660 {
5661 	struct i40e_virtchnl_version_info reply;
5662 
5663 	if (msg_size != sizeof(struct i40e_virtchnl_version_info)) {
5664 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_VERSION,
5665 		    I40E_ERR_PARAM);
5666 		return;
5667 	}
5668 
5669 	reply.major = I40E_VIRTCHNL_VERSION_MAJOR;
5670 	reply.minor = I40E_VIRTCHNL_VERSION_MINOR;
5671 	ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_VERSION, I40E_SUCCESS, &reply,
5672 	    sizeof(reply));
5673 }
5674 
5675 static void
5676 ixl_vf_reset_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5677     uint16_t msg_size)
5678 {
5679 
5680 	if (msg_size != 0) {
5681 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_RESET_VF,
5682 		    I40E_ERR_PARAM);
5683 		return;
5684 	}
5685 
5686 	ixl_reset_vf(pf, vf);
5687 
5688 	/* No response to a reset message. */
5689 }
5690 
5691 static void
5692 ixl_vf_get_resources_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5693     uint16_t msg_size)
5694 {
5695 	struct i40e_virtchnl_vf_resource reply;
5696 
5697 	if (msg_size != 0) {
5698 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
5699 		    I40E_ERR_PARAM);
5700 		return;
5701 	}
5702 
5703 	bzero(&reply, sizeof(reply));
5704 
5705 	reply.vf_offload_flags = I40E_VIRTCHNL_VF_OFFLOAD_L2;
5706 
5707 	reply.num_vsis = 1;
5708 	reply.num_queue_pairs = vf->vsi.num_queues;
5709 	reply.max_vectors = pf->hw.func_caps.num_msix_vectors_vf;
5710 	reply.vsi_res[0].vsi_id = vf->vsi.vsi_num;
5711 	reply.vsi_res[0].vsi_type = I40E_VSI_SRIOV;
5712 	reply.vsi_res[0].num_queue_pairs = vf->vsi.num_queues;
5713 	memcpy(reply.vsi_res[0].default_mac_addr, vf->mac, ETHER_ADDR_LEN);
5714 
5715 	ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
5716 	    I40E_SUCCESS, &reply, sizeof(reply));
5717 }
5718 
5719 static int
5720 ixl_vf_config_tx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
5721     struct i40e_virtchnl_txq_info *info)
5722 {
5723 	struct i40e_hw *hw;
5724 	struct i40e_hmc_obj_txq txq;
5725 	uint16_t global_queue_num, global_vf_num;
5726 	enum i40e_status_code status;
5727 	uint32_t qtx_ctl;
5728 
5729 	hw = &pf->hw;
5730 	global_queue_num = vf->vsi.first_queue + info->queue_id;
5731 	global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
5732 	bzero(&txq, sizeof(txq));
5733 
5734 	status = i40e_clear_lan_tx_queue_context(hw, global_queue_num);
5735 	if (status != I40E_SUCCESS)
5736 		return (EINVAL);
5737 
5738 	txq.base = info->dma_ring_addr / IXL_TX_CTX_BASE_UNITS;
5739 
5740 	txq.head_wb_ena = info->headwb_enabled;
5741 	txq.head_wb_addr = info->dma_headwb_addr;
5742 	txq.qlen = info->ring_len;
5743 	txq.rdylist = le16_to_cpu(vf->vsi.info.qs_handle[0]);
5744 	txq.rdylist_act = 0;
5745 
5746 	status = i40e_set_lan_tx_queue_context(hw, global_queue_num, &txq);
5747 	if (status != I40E_SUCCESS)
5748 		return (EINVAL);
5749 
5750 	qtx_ctl = I40E_QTX_CTL_VF_QUEUE |
5751 	    (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) |
5752 	    (global_vf_num << I40E_QTX_CTL_VFVM_INDX_SHIFT);
5753 	wr32(hw, I40E_QTX_CTL(global_queue_num), qtx_ctl);
5754 	ixl_flush(hw);
5755 
5756 	return (0);
5757 }
5758 
5759 static int
5760 ixl_vf_config_rx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
5761     struct i40e_virtchnl_rxq_info *info)
5762 {
5763 	struct i40e_hw *hw;
5764 	struct i40e_hmc_obj_rxq rxq;
5765 	uint16_t global_queue_num;
5766 	enum i40e_status_code status;
5767 
5768 	hw = &pf->hw;
5769 	global_queue_num = vf->vsi.first_queue + info->queue_id;
5770 	bzero(&rxq, sizeof(rxq));
5771 
5772 	if (info->databuffer_size > IXL_VF_MAX_BUFFER)
5773 		return (EINVAL);
5774 
5775 	if (info->max_pkt_size > IXL_VF_MAX_FRAME ||
5776 	    info->max_pkt_size < ETHER_MIN_LEN)
5777 		return (EINVAL);
5778 
5779 	if (info->splithdr_enabled) {
5780 		if (info->hdr_size > IXL_VF_MAX_HDR_BUFFER)
5781 			return (EINVAL);
5782 
5783 		rxq.hsplit_0 = info->rx_split_pos &
5784 		    (I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_L2 |
5785 		     I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_IP |
5786 		     I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_TCP_UDP |
5787 		     I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_SCTP);
5788 		rxq.hbuff = info->hdr_size >> I40E_RXQ_CTX_HBUFF_SHIFT;
5789 
5790 		rxq.dtype = 2;
5791 	}
5792 
5793 	status = i40e_clear_lan_rx_queue_context(hw, global_queue_num);
5794 	if (status != I40E_SUCCESS)
5795 		return (EINVAL);
5796 
5797 	rxq.base = info->dma_ring_addr / IXL_RX_CTX_BASE_UNITS;
5798 	rxq.qlen = info->ring_len;
5799 
5800 	rxq.dbuff = info->databuffer_size >> I40E_RXQ_CTX_DBUFF_SHIFT;
5801 
5802 	rxq.dsize = 1;
5803 	rxq.crcstrip = 1;
5804 	rxq.l2tsel = 1;
5805 
5806 	rxq.rxmax = info->max_pkt_size;
5807 	rxq.tphrdesc_ena = 1;
5808 	rxq.tphwdesc_ena = 1;
5809 	rxq.tphdata_ena = 1;
5810 	rxq.tphhead_ena = 1;
5811 	rxq.lrxqthresh = 2;
5812 	rxq.prefena = 1;
5813 
5814 	status = i40e_set_lan_rx_queue_context(hw, global_queue_num, &rxq);
5815 	if (status != I40E_SUCCESS)
5816 		return (EINVAL);
5817 
5818 	return (0);
5819 }
5820 
5821 static void
5822 ixl_vf_config_vsi_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5823     uint16_t msg_size)
5824 {
5825 	struct i40e_virtchnl_vsi_queue_config_info *info;
5826 	struct i40e_virtchnl_queue_pair_info *pair;
5827 	int i;
5828 
5829 	if (msg_size < sizeof(*info)) {
5830 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5831 		    I40E_ERR_PARAM);
5832 		return;
5833 	}
5834 
5835 	info = msg;
5836 	if (info->num_queue_pairs == 0) {
5837 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5838 		    I40E_ERR_PARAM);
5839 		return;
5840 	}
5841 
5842 	if (msg_size != sizeof(*info) + info->num_queue_pairs * sizeof(*pair)) {
5843 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5844 		    I40E_ERR_PARAM);
5845 		return;
5846 	}
5847 
5848 	if (info->vsi_id != vf->vsi.vsi_num) {
5849 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5850 		    I40E_ERR_PARAM);
5851 		return;
5852 	}
5853 
5854 	for (i = 0; i < info->num_queue_pairs; i++) {
5855 		pair = &info->qpair[i];
5856 
5857 		if (pair->txq.vsi_id != vf->vsi.vsi_num ||
5858 		    pair->rxq.vsi_id != vf->vsi.vsi_num ||
5859 		    pair->txq.queue_id != pair->rxq.queue_id ||
5860 		    pair->txq.queue_id >= vf->vsi.num_queues) {
5861 
5862 			i40e_send_vf_nack(pf, vf,
5863 			    I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
5864 			return;
5865 		}
5866 
5867 		if (ixl_vf_config_tx_queue(pf, vf, &pair->txq) != 0) {
5868 			i40e_send_vf_nack(pf, vf,
5869 			    I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
5870 			return;
5871 		}
5872 
5873 		if (ixl_vf_config_rx_queue(pf, vf, &pair->rxq) != 0) {
5874 			i40e_send_vf_nack(pf, vf,
5875 			    I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
5876 			return;
5877 		}
5878 	}
5879 
5880 	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES);
5881 }
5882 
5883 static void
5884 ixl_vf_set_qctl(struct ixl_pf *pf,
5885     const struct i40e_virtchnl_vector_map *vector,
5886     enum i40e_queue_type cur_type, uint16_t cur_queue,
5887     enum i40e_queue_type *last_type, uint16_t *last_queue)
5888 {
5889 	uint32_t offset, qctl;
5890 	uint16_t itr_indx;
5891 
5892 	if (cur_type == I40E_QUEUE_TYPE_RX) {
5893 		offset = I40E_QINT_RQCTL(cur_queue);
5894 		itr_indx = vector->rxitr_idx;
5895 	} else {
5896 		offset = I40E_QINT_TQCTL(cur_queue);
5897 		itr_indx = vector->txitr_idx;
5898 	}
5899 
5900 	qctl = htole32((vector->vector_id << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
5901 	    (*last_type << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
5902 	    (*last_queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
5903 	    I40E_QINT_RQCTL_CAUSE_ENA_MASK |
5904 	    (itr_indx << I40E_QINT_RQCTL_ITR_INDX_SHIFT));
5905 
5906 	wr32(&pf->hw, offset, qctl);
5907 
5908 	*last_type = cur_type;
5909 	*last_queue = cur_queue;
5910 }
5911 
5912 static void
5913 ixl_vf_config_vector(struct ixl_pf *pf, struct ixl_vf *vf,
5914     const struct i40e_virtchnl_vector_map *vector)
5915 {
5916 	struct i40e_hw *hw;
5917 	u_int qindex;
5918 	enum i40e_queue_type type, last_type;
5919 	uint32_t lnklst_reg;
5920 	uint16_t rxq_map, txq_map, cur_queue, last_queue;
5921 
5922 	hw = &pf->hw;
5923 
5924 	rxq_map = vector->rxq_map;
5925 	txq_map = vector->txq_map;
5926 
5927 	last_queue = IXL_END_OF_INTR_LNKLST;
5928 	last_type = I40E_QUEUE_TYPE_RX;
5929 
5930 	/*
5931 	 * The datasheet says to optimize performance, RX queues and TX queues
5932 	 * should be interleaved in the interrupt linked list, so we process
5933 	 * both at once here.
5934 	 */
5935 	while ((rxq_map != 0) || (txq_map != 0)) {
5936 		if (txq_map != 0) {
5937 			qindex = ffs(txq_map) - 1;
5938 			type = I40E_QUEUE_TYPE_TX;
5939 			cur_queue = vf->vsi.first_queue + qindex;
5940 			ixl_vf_set_qctl(pf, vector, type, cur_queue,
5941 			    &last_type, &last_queue);
5942 			txq_map &= ~(1 << qindex);
5943 		}
5944 
5945 		if (rxq_map != 0) {
5946 			qindex = ffs(rxq_map) - 1;
5947 			type = I40E_QUEUE_TYPE_RX;
5948 			cur_queue = vf->vsi.first_queue + qindex;
5949 			ixl_vf_set_qctl(pf, vector, type, cur_queue,
5950 			    &last_type, &last_queue);
5951 			rxq_map &= ~(1 << qindex);
5952 		}
5953 	}
5954 
5955 	if (vector->vector_id == 0)
5956 		lnklst_reg = I40E_VPINT_LNKLST0(vf->vf_num);
5957 	else
5958 		lnklst_reg = IXL_VPINT_LNKLSTN_REG(hw, vector->vector_id,
5959 		    vf->vf_num);
5960 	wr32(hw, lnklst_reg,
5961 	    (last_queue << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
5962 	    (last_type << I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
5963 
5964 	ixl_flush(hw);
5965 }
5966 
5967 static void
5968 ixl_vf_config_irq_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5969     uint16_t msg_size)
5970 {
5971 	struct i40e_virtchnl_irq_map_info *map;
5972 	struct i40e_virtchnl_vector_map *vector;
5973 	struct i40e_hw *hw;
5974 	int i, largest_txq, largest_rxq;
5975 
5976 	hw = &pf->hw;
5977 
5978 	if (msg_size < sizeof(*map)) {
5979 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5980 		    I40E_ERR_PARAM);
5981 		return;
5982 	}
5983 
5984 	map = msg;
5985 	if (map->num_vectors == 0) {
5986 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5987 		    I40E_ERR_PARAM);
5988 		return;
5989 	}
5990 
5991 	if (msg_size != sizeof(*map) + map->num_vectors * sizeof(*vector)) {
5992 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5993 		    I40E_ERR_PARAM);
5994 		return;
5995 	}
5996 
5997 	for (i = 0; i < map->num_vectors; i++) {
5998 		vector = &map->vecmap[i];
5999 
6000 		if ((vector->vector_id >= hw->func_caps.num_msix_vectors_vf) ||
6001 		    vector->vsi_id != vf->vsi.vsi_num) {
6002 			i40e_send_vf_nack(pf, vf,
6003 			    I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_ERR_PARAM);
6004 			return;
6005 		}
6006 
6007 		if (vector->rxq_map != 0) {
6008 			largest_rxq = fls(vector->rxq_map) - 1;
6009 			if (largest_rxq >= vf->vsi.num_queues) {
6010 				i40e_send_vf_nack(pf, vf,
6011 				    I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6012 				    I40E_ERR_PARAM);
6013 				return;
6014 			}
6015 		}
6016 
6017 		if (vector->txq_map != 0) {
6018 			largest_txq = fls(vector->txq_map) - 1;
6019 			if (largest_txq >= vf->vsi.num_queues) {
6020 				i40e_send_vf_nack(pf, vf,
6021 				    I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6022 				    I40E_ERR_PARAM);
6023 				return;
6024 			}
6025 		}
6026 
6027 		if (vector->rxitr_idx > IXL_MAX_ITR_IDX ||
6028 		    vector->txitr_idx > IXL_MAX_ITR_IDX) {
6029 			i40e_send_vf_nack(pf, vf,
6030 			    I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6031 			    I40E_ERR_PARAM);
6032 			return;
6033 		}
6034 
6035 		ixl_vf_config_vector(pf, vf, vector);
6036 	}
6037 
6038 	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP);
6039 }
6040 
6041 static void
6042 ixl_vf_enable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6043     uint16_t msg_size)
6044 {
6045 	struct i40e_virtchnl_queue_select *select;
6046 	int error;
6047 
6048 	if (msg_size != sizeof(*select)) {
6049 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6050 		    I40E_ERR_PARAM);
6051 		return;
6052 	}
6053 
6054 	select = msg;
6055 	if (select->vsi_id != vf->vsi.vsi_num ||
6056 	    select->rx_queues == 0 || select->tx_queues == 0) {
6057 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6058 		    I40E_ERR_PARAM);
6059 		return;
6060 	}
6061 
6062 	error = ixl_enable_rings(&vf->vsi);
6063 	if (error) {
6064 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6065 		    I40E_ERR_TIMEOUT);
6066 		return;
6067 	}
6068 
6069 	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES);
6070 }
6071 
6072 static void
6073 ixl_vf_disable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf,
6074     void *msg, uint16_t msg_size)
6075 {
6076 	struct i40e_virtchnl_queue_select *select;
6077 	int error;
6078 
6079 	if (msg_size != sizeof(*select)) {
6080 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6081 		    I40E_ERR_PARAM);
6082 		return;
6083 	}
6084 
6085 	select = msg;
6086 	if (select->vsi_id != vf->vsi.vsi_num ||
6087 	    select->rx_queues == 0 || select->tx_queues == 0) {
6088 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6089 		    I40E_ERR_PARAM);
6090 		return;
6091 	}
6092 
6093 	error = ixl_disable_rings(&vf->vsi);
6094 	if (error) {
6095 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6096 		    I40E_ERR_TIMEOUT);
6097 		return;
6098 	}
6099 
6100 	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES);
6101 }
6102 
6103 static boolean_t
6104 ixl_zero_mac(const uint8_t *addr)
6105 {
6106 	uint8_t zero[ETHER_ADDR_LEN] = {0, 0, 0, 0, 0, 0};
6107 
6108 	return (cmp_etheraddr(addr, zero));
6109 }
6110 
6111 static boolean_t
6112 ixl_bcast_mac(const uint8_t *addr)
6113 {
6114 
6115 	return (cmp_etheraddr(addr, ixl_bcast_addr));
6116 }
6117 
6118 static int
6119 ixl_vf_mac_valid(struct ixl_vf *vf, const uint8_t *addr)
6120 {
6121 
6122 	if (ixl_zero_mac(addr) || ixl_bcast_mac(addr))
6123 		return (EINVAL);
6124 
6125 	/*
6126 	 * If the VF is not allowed to change its MAC address, don't let it
6127 	 * set a MAC filter for an address that is not a multicast address and
6128 	 * is not its assigned MAC.
6129 	 */
6130 	if (!(vf->vf_flags & VF_FLAG_SET_MAC_CAP) &&
6131 	    !(ETHER_IS_MULTICAST(addr) || cmp_etheraddr(addr, vf->mac)))
6132 		return (EPERM);
6133 
6134 	return (0);
6135 }
6136 
6137 static void
6138 ixl_vf_add_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6139     uint16_t msg_size)
6140 {
6141 	struct i40e_virtchnl_ether_addr_list *addr_list;
6142 	struct i40e_virtchnl_ether_addr *addr;
6143 	struct ixl_vsi *vsi;
6144 	int i;
6145 	size_t expected_size;
6146 
6147 	vsi = &vf->vsi;
6148 
6149 	if (msg_size < sizeof(*addr_list)) {
6150 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6151 		    I40E_ERR_PARAM);
6152 		return;
6153 	}
6154 
6155 	addr_list = msg;
6156 	expected_size = sizeof(*addr_list) +
6157 	    addr_list->num_elements * sizeof(*addr);
6158 
6159 	if (addr_list->num_elements == 0 ||
6160 	    addr_list->vsi_id != vsi->vsi_num ||
6161 	    msg_size != expected_size) {
6162 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6163 		    I40E_ERR_PARAM);
6164 		return;
6165 	}
6166 
6167 	for (i = 0; i < addr_list->num_elements; i++) {
6168 		if (ixl_vf_mac_valid(vf, addr_list->list[i].addr) != 0) {
6169 			i40e_send_vf_nack(pf, vf,
6170 			    I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6171 			return;
6172 		}
6173 	}
6174 
6175 	for (i = 0; i < addr_list->num_elements; i++) {
6176 		addr = &addr_list->list[i];
6177 		ixl_add_filter(vsi, addr->addr, IXL_VLAN_ANY);
6178 	}
6179 
6180 	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS);
6181 }
6182 
6183 static void
6184 ixl_vf_del_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6185     uint16_t msg_size)
6186 {
6187 	struct i40e_virtchnl_ether_addr_list *addr_list;
6188 	struct i40e_virtchnl_ether_addr *addr;
6189 	size_t expected_size;
6190 	int i;
6191 
6192 	if (msg_size < sizeof(*addr_list)) {
6193 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6194 		    I40E_ERR_PARAM);
6195 		return;
6196 	}
6197 
6198 	addr_list = msg;
6199 	expected_size = sizeof(*addr_list) +
6200 	    addr_list->num_elements * sizeof(*addr);
6201 
6202 	if (addr_list->num_elements == 0 ||
6203 	    addr_list->vsi_id != vf->vsi.vsi_num ||
6204 	    msg_size != expected_size) {
6205 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6206 		    I40E_ERR_PARAM);
6207 		return;
6208 	}
6209 
6210 	for (i = 0; i < addr_list->num_elements; i++) {
6211 		addr = &addr_list->list[i];
6212 		if (ixl_zero_mac(addr->addr) || ixl_bcast_mac(addr->addr)) {
6213 			i40e_send_vf_nack(pf, vf,
6214 			    I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6215 			return;
6216 		}
6217 	}
6218 
6219 	for (i = 0; i < addr_list->num_elements; i++) {
6220 		addr = &addr_list->list[i];
6221 		ixl_del_filter(&vf->vsi, addr->addr, IXL_VLAN_ANY);
6222 	}
6223 
6224 	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS);
6225 }
6226 
6227 static enum i40e_status_code
6228 ixl_vf_enable_vlan_strip(struct ixl_pf *pf, struct ixl_vf *vf)
6229 {
6230 	struct i40e_vsi_context vsi_ctx;
6231 
6232 	vsi_ctx.seid = vf->vsi.seid;
6233 
6234 	bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
6235 	vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
6236 	vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
6237 	    I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
6238 	return (i40e_aq_update_vsi_params(&pf->hw, &vsi_ctx, NULL));
6239 }
6240 
6241 static void
6242 ixl_vf_add_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6243     uint16_t msg_size)
6244 {
6245 	struct i40e_virtchnl_vlan_filter_list *filter_list;
6246 	enum i40e_status_code code;
6247 	size_t expected_size;
6248 	int i;
6249 
6250 	if (msg_size < sizeof(*filter_list)) {
6251 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6252 		    I40E_ERR_PARAM);
6253 		return;
6254 	}
6255 
6256 	filter_list = msg;
6257 	expected_size = sizeof(*filter_list) +
6258 	    filter_list->num_elements * sizeof(uint16_t);
6259 	if (filter_list->num_elements == 0 ||
6260 	    filter_list->vsi_id != vf->vsi.vsi_num ||
6261 	    msg_size != expected_size) {
6262 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6263 		    I40E_ERR_PARAM);
6264 		return;
6265 	}
6266 
6267 	if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
6268 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6269 		    I40E_ERR_PARAM);
6270 		return;
6271 	}
6272 
6273 	for (i = 0; i < filter_list->num_elements; i++) {
6274 		if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
6275 			i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6276 			    I40E_ERR_PARAM);
6277 			return;
6278 		}
6279 	}
6280 
6281 	code = ixl_vf_enable_vlan_strip(pf, vf);
6282 	if (code != I40E_SUCCESS) {
6283 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6284 		    I40E_ERR_PARAM);
6285 	}
6286 
6287 	for (i = 0; i < filter_list->num_elements; i++)
6288 		ixl_add_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
6289 
6290 	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN);
6291 }
6292 
6293 static void
6294 ixl_vf_del_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6295     uint16_t msg_size)
6296 {
6297 	struct i40e_virtchnl_vlan_filter_list *filter_list;
6298 	int i;
6299 	size_t expected_size;
6300 
6301 	if (msg_size < sizeof(*filter_list)) {
6302 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
6303 		    I40E_ERR_PARAM);
6304 		return;
6305 	}
6306 
6307 	filter_list = msg;
6308 	expected_size = sizeof(*filter_list) +
6309 	    filter_list->num_elements * sizeof(uint16_t);
6310 	if (filter_list->num_elements == 0 ||
6311 	    filter_list->vsi_id != vf->vsi.vsi_num ||
6312 	    msg_size != expected_size) {
6313 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
6314 		    I40E_ERR_PARAM);
6315 		return;
6316 	}
6317 
6318 	for (i = 0; i < filter_list->num_elements; i++) {
6319 		if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
6320 			i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6321 			    I40E_ERR_PARAM);
6322 			return;
6323 		}
6324 	}
6325 
6326 	if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
6327 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6328 		    I40E_ERR_PARAM);
6329 		return;
6330 	}
6331 
6332 	for (i = 0; i < filter_list->num_elements; i++)
6333 		ixl_del_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
6334 
6335 	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN);
6336 }
6337 
6338 static void
6339 ixl_vf_config_promisc_msg(struct ixl_pf *pf, struct ixl_vf *vf,
6340     void *msg, uint16_t msg_size)
6341 {
6342 	struct i40e_virtchnl_promisc_info *info;
6343 	enum i40e_status_code code;
6344 
6345 	if (msg_size != sizeof(*info)) {
6346 		i40e_send_vf_nack(pf, vf,
6347 		    I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
6348 		return;
6349 	}
6350 
6351 	if (!(vf->vf_flags & VF_FLAG_PROMISC_CAP)) {
6352 		i40e_send_vf_nack(pf, vf,
6353 		    I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
6354 		return;
6355 	}
6356 
6357 	info = msg;
6358 	if (info->vsi_id != vf->vsi.vsi_num) {
6359 		i40e_send_vf_nack(pf, vf,
6360 		    I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
6361 		return;
6362 	}
6363 
6364 	code = i40e_aq_set_vsi_unicast_promiscuous(&pf->hw, info->vsi_id,
6365 	    info->flags & I40E_FLAG_VF_UNICAST_PROMISC, NULL);
6366 	if (code != I40E_SUCCESS) {
6367 		i40e_send_vf_nack(pf, vf,
6368 		    I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
6369 		return;
6370 	}
6371 
6372 	code = i40e_aq_set_vsi_multicast_promiscuous(&pf->hw, info->vsi_id,
6373 	    info->flags & I40E_FLAG_VF_MULTICAST_PROMISC, NULL);
6374 	if (code != I40E_SUCCESS) {
6375 		i40e_send_vf_nack(pf, vf,
6376 		    I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
6377 		return;
6378 	}
6379 
6380 	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE);
6381 }
6382 
6383 static void
6384 ixl_vf_get_stats_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6385     uint16_t msg_size)
6386 {
6387 	struct i40e_virtchnl_queue_select *queue;
6388 
6389 	if (msg_size != sizeof(*queue)) {
6390 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
6391 		    I40E_ERR_PARAM);
6392 		return;
6393 	}
6394 
6395 	queue = msg;
6396 	if (queue->vsi_id != vf->vsi.vsi_num) {
6397 		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
6398 		    I40E_ERR_PARAM);
6399 		return;
6400 	}
6401 
6402 	ixl_update_eth_stats(&vf->vsi);
6403 
6404 	ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
6405 	    I40E_SUCCESS, &vf->vsi.eth_stats, sizeof(vf->vsi.eth_stats));
6406 }
6407 
6408 static void
6409 ixl_handle_vf_msg(struct ixl_pf *pf, struct i40e_arq_event_info *event)
6410 {
6411 	struct ixl_vf *vf;
6412 	void *msg;
6413 	uint16_t vf_num, msg_size;
6414 	uint32_t opcode;
6415 
6416 	vf_num = le16toh(event->desc.retval) - pf->hw.func_caps.vf_base_id;
6417 	opcode = le32toh(event->desc.cookie_high);
6418 
6419 	if (vf_num >= pf->num_vfs) {
6420 		device_printf(pf->dev, "Got msg from illegal VF: %d\n", vf_num);
6421 		return;
6422 	}
6423 
6424 	vf = &pf->vfs[vf_num];
6425 	msg = event->msg_buf;
6426 	msg_size = event->msg_len;
6427 
6428 	I40E_VC_DEBUG(pf, ixl_vc_opcode_level(opcode),
6429 	    "Got msg %s(%d) from VF-%d of size %d\n",
6430 	    ixl_vc_opcode_str(opcode), opcode, vf_num, msg_size);
6431 
6432 	switch (opcode) {
6433 	case I40E_VIRTCHNL_OP_VERSION:
6434 		ixl_vf_version_msg(pf, vf, msg, msg_size);
6435 		break;
6436 	case I40E_VIRTCHNL_OP_RESET_VF:
6437 		ixl_vf_reset_msg(pf, vf, msg, msg_size);
6438 		break;
6439 	case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
6440 		ixl_vf_get_resources_msg(pf, vf, msg, msg_size);
6441 		break;
6442 	case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
6443 		ixl_vf_config_vsi_msg(pf, vf, msg, msg_size);
6444 		break;
6445 	case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
6446 		ixl_vf_config_irq_msg(pf, vf, msg, msg_size);
6447 		break;
6448 	case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
6449 		ixl_vf_enable_queues_msg(pf, vf, msg, msg_size);
6450 		break;
6451 	case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
6452 		ixl_vf_disable_queues_msg(pf, vf, msg, msg_size);
6453 		break;
6454 	case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
6455 		ixl_vf_add_mac_msg(pf, vf, msg, msg_size);
6456 		break;
6457 	case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
6458 		ixl_vf_del_mac_msg(pf, vf, msg, msg_size);
6459 		break;
6460 	case I40E_VIRTCHNL_OP_ADD_VLAN:
6461 		ixl_vf_add_vlan_msg(pf, vf, msg, msg_size);
6462 		break;
6463 	case I40E_VIRTCHNL_OP_DEL_VLAN:
6464 		ixl_vf_del_vlan_msg(pf, vf, msg, msg_size);
6465 		break;
6466 	case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
6467 		ixl_vf_config_promisc_msg(pf, vf, msg, msg_size);
6468 		break;
6469 	case I40E_VIRTCHNL_OP_GET_STATS:
6470 		ixl_vf_get_stats_msg(pf, vf, msg, msg_size);
6471 		break;
6472 
6473 	/* These two opcodes have been superseded by CONFIG_VSI_QUEUES. */
6474 	case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
6475 	case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
6476 	default:
6477 		i40e_send_vf_nack(pf, vf, opcode, I40E_ERR_NOT_IMPLEMENTED);
6478 		break;
6479 	}
6480 }
6481 
6482 /* Handle any VFs that have reset themselves via a Function Level Reset(FLR). */
6483 static void
6484 ixl_handle_vflr(void *arg, int pending)
6485 {
6486 	struct ixl_pf *pf;
6487 	struct i40e_hw *hw;
6488 	uint16_t global_vf_num;
6489 	uint32_t vflrstat_index, vflrstat_mask, vflrstat, icr0;
6490 	int i;
6491 
6492 	pf = arg;
6493 	hw = &pf->hw;
6494 
6495 	IXL_PF_LOCK(pf);
6496 	for (i = 0; i < pf->num_vfs; i++) {
6497 		global_vf_num = hw->func_caps.vf_base_id + i;
6498 
6499 		vflrstat_index = IXL_GLGEN_VFLRSTAT_INDEX(global_vf_num);
6500 		vflrstat_mask = IXL_GLGEN_VFLRSTAT_MASK(global_vf_num);
6501 		vflrstat = rd32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index));
6502 		if (vflrstat & vflrstat_mask) {
6503 			wr32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index),
6504 			    vflrstat_mask);
6505 
6506 			ixl_reinit_vf(pf, &pf->vfs[i]);
6507 		}
6508 	}
6509 
6510 	icr0 = rd32(hw, I40E_PFINT_ICR0_ENA);
6511 	icr0 |= I40E_PFINT_ICR0_ENA_VFLR_MASK;
6512 	wr32(hw, I40E_PFINT_ICR0_ENA, icr0);
6513 	ixl_flush(hw);
6514 
6515 	IXL_PF_UNLOCK(pf);
6516 }
6517 
6518 static int
6519 ixl_adminq_err_to_errno(enum i40e_admin_queue_err err)
6520 {
6521 
6522 	switch (err) {
6523 	case I40E_AQ_RC_EPERM:
6524 		return (EPERM);
6525 	case I40E_AQ_RC_ENOENT:
6526 		return (ENOENT);
6527 	case I40E_AQ_RC_ESRCH:
6528 		return (ESRCH);
6529 	case I40E_AQ_RC_EINTR:
6530 		return (EINTR);
6531 	case I40E_AQ_RC_EIO:
6532 		return (EIO);
6533 	case I40E_AQ_RC_ENXIO:
6534 		return (ENXIO);
6535 	case I40E_AQ_RC_E2BIG:
6536 		return (E2BIG);
6537 	case I40E_AQ_RC_EAGAIN:
6538 		return (EAGAIN);
6539 	case I40E_AQ_RC_ENOMEM:
6540 		return (ENOMEM);
6541 	case I40E_AQ_RC_EACCES:
6542 		return (EACCES);
6543 	case I40E_AQ_RC_EFAULT:
6544 		return (EFAULT);
6545 	case I40E_AQ_RC_EBUSY:
6546 		return (EBUSY);
6547 	case I40E_AQ_RC_EEXIST:
6548 		return (EEXIST);
6549 	case I40E_AQ_RC_EINVAL:
6550 		return (EINVAL);
6551 	case I40E_AQ_RC_ENOTTY:
6552 		return (ENOTTY);
6553 	case I40E_AQ_RC_ENOSPC:
6554 		return (ENOSPC);
6555 	case I40E_AQ_RC_ENOSYS:
6556 		return (ENOSYS);
6557 	case I40E_AQ_RC_ERANGE:
6558 		return (ERANGE);
6559 	case I40E_AQ_RC_EFLUSHED:
6560 		return (EINVAL);	/* No exact equivalent in errno.h */
6561 	case I40E_AQ_RC_BAD_ADDR:
6562 		return (EFAULT);
6563 	case I40E_AQ_RC_EMODE:
6564 		return (EPERM);
6565 	case I40E_AQ_RC_EFBIG:
6566 		return (EFBIG);
6567 	default:
6568 		return (EINVAL);
6569 	}
6570 }
6571 
6572 static int
6573 ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params)
6574 {
6575 	struct ixl_pf *pf;
6576 	struct i40e_hw *hw;
6577 	struct ixl_vsi *pf_vsi;
6578 	enum i40e_status_code ret;
6579 	int i, error;
6580 
6581 	pf = device_get_softc(dev);
6582 	hw = &pf->hw;
6583 	pf_vsi = &pf->vsi;
6584 
6585 	IXL_PF_LOCK(pf);
6586 	pf->vfs = malloc(sizeof(struct ixl_vf) * num_vfs, M_IXL, M_NOWAIT |
6587 	    M_ZERO);
6588 
6589 	if (pf->vfs == NULL) {
6590 		error = ENOMEM;
6591 		goto fail;
6592 	}
6593 
6594 	for (i = 0; i < num_vfs; i++)
6595 		sysctl_ctx_init(&pf->vfs[i].ctx);
6596 
6597 	ret = i40e_aq_add_veb(hw, pf_vsi->uplink_seid, pf_vsi->seid,
6598 	    1, FALSE, FALSE, &pf->veb_seid, NULL);
6599 	if (ret != I40E_SUCCESS) {
6600 		error = ixl_adminq_err_to_errno(hw->aq.asq_last_status);
6601 		device_printf(dev, "add_veb failed; code=%d error=%d", ret,
6602 		    error);
6603 		goto fail;
6604 	}
6605 
6606 	ixl_configure_msix(pf);
6607 	ixl_enable_adminq(hw);
6608 
6609 	pf->num_vfs = num_vfs;
6610 	IXL_PF_UNLOCK(pf);
6611 	return (0);
6612 
6613 fail:
6614 	free(pf->vfs, M_IXL);
6615 	pf->vfs = NULL;
6616 	IXL_PF_UNLOCK(pf);
6617 	return (error);
6618 }
6619 
6620 static void
6621 ixl_iov_uninit(device_t dev)
6622 {
6623 	struct ixl_pf *pf;
6624 	struct i40e_hw *hw;
6625 	struct ixl_vsi *vsi;
6626 	struct ifnet *ifp;
6627 	struct ixl_vf *vfs;
6628 	int i, num_vfs;
6629 
6630 	pf = device_get_softc(dev);
6631 	hw = &pf->hw;
6632 	vsi = &pf->vsi;
6633 	ifp = vsi->ifp;
6634 
6635 	IXL_PF_LOCK(pf);
6636 	for (i = 0; i < pf->num_vfs; i++) {
6637 		if (pf->vfs[i].vsi.seid != 0)
6638 			i40e_aq_delete_element(hw, pf->vfs[i].vsi.seid, NULL);
6639 	}
6640 
6641 	if (pf->veb_seid != 0) {
6642 		i40e_aq_delete_element(hw, pf->veb_seid, NULL);
6643 		pf->veb_seid = 0;
6644 	}
6645 
6646 #if __FreeBSD_version > 1100022
6647 	if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
6648 #else
6649 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
6650 #endif
6651 		ixl_disable_intr(vsi);
6652 
6653 	vfs = pf->vfs;
6654 	num_vfs = pf->num_vfs;
6655 
6656 	pf->vfs = NULL;
6657 	pf->num_vfs = 0;
6658 	IXL_PF_UNLOCK(pf);
6659 
6660 	/* Do this after the unlock as sysctl_ctx_free might sleep. */
6661 	for (i = 0; i < num_vfs; i++)
6662 		sysctl_ctx_free(&vfs[i].ctx);
6663 	free(vfs, M_IXL);
6664 }
6665 
6666 static int
6667 ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params)
6668 {
6669 	char sysctl_name[QUEUE_NAME_LEN];
6670 	struct ixl_pf *pf;
6671 	struct ixl_vf *vf;
6672 	const void *mac;
6673 	size_t size;
6674 	int error;
6675 
6676 	pf = device_get_softc(dev);
6677 	vf = &pf->vfs[vfnum];
6678 
6679 	IXL_PF_LOCK(pf);
6680 	vf->vf_num = vfnum;
6681 
6682 	vf->vsi.back = pf;
6683 	vf->vf_flags = VF_FLAG_ENABLED;
6684 	SLIST_INIT(&vf->vsi.ftl);
6685 
6686 	error = ixl_vf_setup_vsi(pf, vf);
6687 	if (error != 0)
6688 		goto out;
6689 
6690 	if (nvlist_exists_binary(params, "mac-addr")) {
6691 		mac = nvlist_get_binary(params, "mac-addr", &size);
6692 		bcopy(mac, vf->mac, ETHER_ADDR_LEN);
6693 
6694 		if (nvlist_get_bool(params, "allow-set-mac"))
6695 			vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
6696 	} else
6697 		/*
6698 		 * If the administrator has not specified a MAC address then
6699 		 * we must allow the VF to choose one.
6700 		 */
6701 		vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
6702 
6703 	if (nvlist_get_bool(params, "mac-anti-spoof"))
6704 		vf->vf_flags |= VF_FLAG_MAC_ANTI_SPOOF;
6705 
6706 	if (nvlist_get_bool(params, "allow-promisc"))
6707 		vf->vf_flags |= VF_FLAG_PROMISC_CAP;
6708 
6709 	vf->vf_flags |= VF_FLAG_VLAN_CAP;
6710 
6711 	ixl_reset_vf(pf, vf);
6712 out:
6713 	IXL_PF_UNLOCK(pf);
6714 	if (error == 0) {
6715 		snprintf(sysctl_name, sizeof(sysctl_name), "vf%d", vfnum);
6716 		ixl_add_vsi_sysctls(pf, &vf->vsi, &vf->ctx, sysctl_name);
6717 	}
6718 
6719 	return (error);
6720 }
6721 #endif /* PCI_IOV */
6722