xref: /freebsd/sys/dev/ixgbe/if_ix.c (revision 9fe48b8076ae9c6dc486d713c468ff03ec056eda)
1 /******************************************************************************
2 
3   Copyright (c) 2001-2015, Intel Corporation
4   All rights reserved.
5 
6   Redistribution and use in source and binary forms, with or without
7   modification, are permitted provided that the following conditions are met:
8 
9    1. Redistributions of source code must retain the above copyright notice,
10       this list of conditions and the following disclaimer.
11 
12    2. Redistributions in binary form must reproduce the above copyright
13       notice, this list of conditions and the following disclaimer in the
14       documentation and/or other materials provided with the distribution.
15 
16    3. Neither the name of the Intel Corporation nor the names of its
17       contributors may be used to endorse or promote products derived from
18       this software without specific prior written permission.
19 
20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30   POSSIBILITY OF SUCH DAMAGE.
31 
32 ******************************************************************************/
33 /*$FreeBSD$*/
34 
35 
36 #ifndef IXGBE_STANDALONE_BUILD
37 #include "opt_inet.h"
38 #include "opt_inet6.h"
39 #include "opt_rss.h"
40 #endif
41 
42 #include "ixgbe.h"
43 
44 #ifdef	RSS
45 #include <net/rss_config.h>
46 #include <netinet/in_rss.h>
47 #endif
48 
49 /*********************************************************************
50  *  Set this to one to display debug statistics
51  *********************************************************************/
52 int             ixgbe_display_debug_stats = 0;
53 
54 /*********************************************************************
55  *  Driver version
56  *********************************************************************/
57 char ixgbe_driver_version[] = "2.7.4";
58 
59 /*********************************************************************
60  *  PCI Device ID Table
61  *
62  *  Used by probe to select devices to load on
63  *  Last field stores an index into ixgbe_strings
64  *  Last entry must be all 0s
65  *
66  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
67  *********************************************************************/
68 
69 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
70 {
71 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
72 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
73 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
74 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
75 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
76 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
77 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
78 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
79 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
80 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
81 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
82 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
83 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
84 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
85 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
86 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
87 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
88 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
89 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
90 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
91 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
92 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
93 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0},
94 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_QSFP_SF_QP, 0, 0, 0},
95 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
96 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
97 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T, 0, 0, 0},
98 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KR, 0, 0, 0},
99 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KX4, 0, 0, 0},
100 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_10G_T, 0, 0, 0},
101 	/* required last entry */
102 	{0, 0, 0, 0, 0}
103 };
104 
105 /*********************************************************************
106  *  Table of branding strings
107  *********************************************************************/
108 
109 static char    *ixgbe_strings[] = {
110 	"Intel(R) PRO/10GbE PCI-Express Network Driver"
111 };
112 
113 /*********************************************************************
114  *  Function prototypes
115  *********************************************************************/
116 static int      ixgbe_probe(device_t);
117 static int      ixgbe_attach(device_t);
118 static int      ixgbe_detach(device_t);
119 static int      ixgbe_shutdown(device_t);
120 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t);
121 static void	ixgbe_init(void *);
122 static void	ixgbe_init_locked(struct adapter *);
123 static void     ixgbe_stop(void *);
124 #if __FreeBSD_version >= 1100036
125 static uint64_t	ixgbe_get_counter(struct ifnet *, ift_counter);
126 #endif
127 static void	ixgbe_add_media_types(struct adapter *);
128 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
129 static int      ixgbe_media_change(struct ifnet *);
130 static void     ixgbe_identify_hardware(struct adapter *);
131 static int      ixgbe_allocate_pci_resources(struct adapter *);
132 static void	ixgbe_get_slot_info(struct ixgbe_hw *);
133 static int      ixgbe_allocate_msix(struct adapter *);
134 static int      ixgbe_allocate_legacy(struct adapter *);
135 static int	ixgbe_setup_msix(struct adapter *);
136 static void	ixgbe_free_pci_resources(struct adapter *);
137 static void	ixgbe_local_timer(void *);
138 static int	ixgbe_setup_interface(device_t, struct adapter *);
139 static void	ixgbe_config_link(struct adapter *);
140 static void	ixgbe_rearm_queues(struct adapter *, u64);
141 
142 static void     ixgbe_initialize_transmit_units(struct adapter *);
143 static void     ixgbe_initialize_receive_units(struct adapter *);
144 static void	ixgbe_enable_rx_drop(struct adapter *);
145 static void	ixgbe_disable_rx_drop(struct adapter *);
146 
147 static void     ixgbe_enable_intr(struct adapter *);
148 static void     ixgbe_disable_intr(struct adapter *);
149 static void     ixgbe_update_stats_counters(struct adapter *);
150 static void     ixgbe_set_promisc(struct adapter *);
151 static void     ixgbe_set_multi(struct adapter *);
152 static void     ixgbe_update_link_status(struct adapter *);
153 static int	ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
154 static int	ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
155 static int	ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
156 static void	ixgbe_set_ivar(struct adapter *, u8, u8, s8);
157 static void	ixgbe_configure_ivars(struct adapter *);
158 static u8 *	ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
159 
160 static void	ixgbe_setup_vlan_hw_support(struct adapter *);
161 static void	ixgbe_register_vlan(void *, struct ifnet *, u16);
162 static void	ixgbe_unregister_vlan(void *, struct ifnet *, u16);
163 
164 static void     ixgbe_add_hw_stats(struct adapter *adapter);
165 
166 /* Support for pluggable optic modules */
167 static bool	ixgbe_sfp_probe(struct adapter *);
168 static void	ixgbe_setup_optics(struct adapter *);
169 
170 /* Legacy (single vector interrupt handler */
171 static void	ixgbe_legacy_irq(void *);
172 
173 /* The MSI/X Interrupt handlers */
174 static void	ixgbe_msix_que(void *);
175 static void	ixgbe_msix_link(void *);
176 
177 /* Deferred interrupt tasklets */
178 static void	ixgbe_handle_que(void *, int);
179 static void	ixgbe_handle_link(void *, int);
180 static void	ixgbe_handle_msf(void *, int);
181 static void	ixgbe_handle_mod(void *, int);
182 
183 #ifdef IXGBE_FDIR
184 static void	ixgbe_reinit_fdir(void *, int);
185 #endif
186 
187 
188 /* Missing shared code prototype */
189 extern void ixgbe_stop_mac_link_on_d3_82599(struct ixgbe_hw *hw);
190 
191 /*********************************************************************
192  *  FreeBSD Device Interface Entry Points
193  *********************************************************************/
194 
195 static device_method_t ix_methods[] = {
196 	/* Device interface */
197 	DEVMETHOD(device_probe, ixgbe_probe),
198 	DEVMETHOD(device_attach, ixgbe_attach),
199 	DEVMETHOD(device_detach, ixgbe_detach),
200 	DEVMETHOD(device_shutdown, ixgbe_shutdown),
201 	DEVMETHOD_END
202 };
203 
204 static driver_t ix_driver = {
205 	"ix", ix_methods, sizeof(struct adapter),
206 };
207 
208 devclass_t ix_devclass;
209 DRIVER_MODULE(ix, pci, ix_driver, ix_devclass, 0, 0);
210 
211 MODULE_DEPEND(ix, pci, 1, 1, 1);
212 MODULE_DEPEND(ix, ether, 1, 1, 1);
213 
214 /*
215 ** TUNEABLE PARAMETERS:
216 */
217 
218 static SYSCTL_NODE(_hw, OID_AUTO, ix, CTLFLAG_RD, 0,
219 		   "IXGBE driver parameters");
220 
221 /*
222 ** AIM: Adaptive Interrupt Moderation
223 ** which means that the interrupt rate
224 ** is varied over time based on the
225 ** traffic for that interrupt vector
226 */
227 static int ixgbe_enable_aim = TRUE;
228 SYSCTL_INT(_hw_ix, OID_AUTO, enable_aim, CTLFLAG_RWTUN, &ixgbe_enable_aim, 0,
229     "Enable adaptive interrupt moderation");
230 
231 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
232 SYSCTL_INT(_hw_ix, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
233     &ixgbe_max_interrupt_rate, 0, "Maximum interrupts per second");
234 
235 /* How many packets rxeof tries to clean at a time */
236 static int ixgbe_rx_process_limit = 256;
237 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
238 SYSCTL_INT(_hw_ix, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
239     &ixgbe_rx_process_limit, 0,
240     "Maximum number of received packets to process at a time,"
241     "-1 means unlimited");
242 
243 /* How many packets txeof tries to clean at a time */
244 static int ixgbe_tx_process_limit = 256;
245 TUNABLE_INT("hw.ixgbe.tx_process_limit", &ixgbe_tx_process_limit);
246 SYSCTL_INT(_hw_ix, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN,
247     &ixgbe_tx_process_limit, 0,
248     "Maximum number of sent packets to process at a time,"
249     "-1 means unlimited");
250 
251 /*
252 ** Smart speed setting, default to on
253 ** this only works as a compile option
254 ** right now as its during attach, set
255 ** this to 'ixgbe_smart_speed_off' to
256 ** disable.
257 */
258 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
259 
260 /*
261  * MSIX should be the default for best performance,
262  * but this allows it to be forced off for testing.
263  */
264 static int ixgbe_enable_msix = 1;
265 SYSCTL_INT(_hw_ix, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixgbe_enable_msix, 0,
266     "Enable MSI-X interrupts");
267 
268 /*
269  * Number of Queues, can be set to 0,
270  * it then autoconfigures based on the
271  * number of cpus with a max of 8. This
272  * can be overriden manually here.
273  */
274 static int ixgbe_num_queues = 0;
275 SYSCTL_INT(_hw_ix, OID_AUTO, num_queues, CTLFLAG_RDTUN, &ixgbe_num_queues, 0,
276     "Number of queues to configure, 0 indicates autoconfigure");
277 
278 /*
279 ** Number of TX descriptors per ring,
280 ** setting higher than RX as this seems
281 ** the better performing choice.
282 */
283 static int ixgbe_txd = PERFORM_TXD;
284 SYSCTL_INT(_hw_ix, OID_AUTO, txd, CTLFLAG_RDTUN, &ixgbe_txd, 0,
285     "Number of transmit descriptors per queue");
286 
287 /* Number of RX descriptors per ring */
288 static int ixgbe_rxd = PERFORM_RXD;
289 SYSCTL_INT(_hw_ix, OID_AUTO, rxd, CTLFLAG_RDTUN, &ixgbe_rxd, 0,
290     "Number of receive descriptors per queue");
291 
292 /*
293 ** Defining this on will allow the use
294 ** of unsupported SFP+ modules, note that
295 ** doing so you are on your own :)
296 */
297 static int allow_unsupported_sfp = FALSE;
298 TUNABLE_INT("hw.ix.unsupported_sfp", &allow_unsupported_sfp);
299 
300 /* Keep running tab on them for sanity check */
301 static int ixgbe_total_ports;
302 
303 #ifdef IXGBE_FDIR
304 /*
305 ** Flow Director actually 'steals'
306 ** part of the packet buffer as its
307 ** filter pool, this variable controls
308 ** how much it uses:
309 **  0 = 64K, 1 = 128K, 2 = 256K
310 */
311 static int fdir_pballoc = 1;
312 #endif
313 
314 #ifdef DEV_NETMAP
315 /*
316  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
317  * be a reference on how to implement netmap support in a driver.
318  * Additional comments are in ixgbe_netmap.h .
319  *
320  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
321  * that extend the standard driver.
322  */
323 #include <dev/netmap/ixgbe_netmap.h>
324 #endif /* DEV_NETMAP */
325 
326 /*********************************************************************
327  *  Device identification routine
328  *
329  *  ixgbe_probe determines if the driver should be loaded on
330  *  adapter based on PCI vendor/device id of the adapter.
331  *
332  *  return BUS_PROBE_DEFAULT on success, positive on failure
333  *********************************************************************/
334 
335 static int
336 ixgbe_probe(device_t dev)
337 {
338 	ixgbe_vendor_info_t *ent;
339 
340 	u16	pci_vendor_id = 0;
341 	u16	pci_device_id = 0;
342 	u16	pci_subvendor_id = 0;
343 	u16	pci_subdevice_id = 0;
344 	char	adapter_name[256];
345 
346 	INIT_DEBUGOUT("ixgbe_probe: begin");
347 
348 	pci_vendor_id = pci_get_vendor(dev);
349 	if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
350 		return (ENXIO);
351 
352 	pci_device_id = pci_get_device(dev);
353 	pci_subvendor_id = pci_get_subvendor(dev);
354 	pci_subdevice_id = pci_get_subdevice(dev);
355 
356 	ent = ixgbe_vendor_info_array;
357 	while (ent->vendor_id != 0) {
358 		if ((pci_vendor_id == ent->vendor_id) &&
359 		    (pci_device_id == ent->device_id) &&
360 
361 		    ((pci_subvendor_id == ent->subvendor_id) ||
362 		     (ent->subvendor_id == 0)) &&
363 
364 		    ((pci_subdevice_id == ent->subdevice_id) ||
365 		     (ent->subdevice_id == 0))) {
366 			sprintf(adapter_name, "%s, Version - %s",
367 				ixgbe_strings[ent->index],
368 				ixgbe_driver_version);
369 			device_set_desc_copy(dev, adapter_name);
370 			++ixgbe_total_ports;
371 			return (BUS_PROBE_DEFAULT);
372 		}
373 		ent++;
374 	}
375 	return (ENXIO);
376 }
377 
378 /*********************************************************************
379  *  Device initialization routine
380  *
381  *  The attach entry point is called when the driver is being loaded.
382  *  This routine identifies the type of hardware, allocates all resources
383  *  and initializes the hardware.
384  *
385  *  return 0 on success, positive on failure
386  *********************************************************************/
387 
388 static int
389 ixgbe_attach(device_t dev)
390 {
391 	struct adapter *adapter;
392 	struct ixgbe_hw *hw;
393 	int             error = 0;
394 	u16		csum;
395 	u32		ctrl_ext;
396 
397 	INIT_DEBUGOUT("ixgbe_attach: begin");
398 
399 	/* Allocate, clear, and link in our adapter structure */
400 	adapter = device_get_softc(dev);
401 	adapter->dev = adapter->osdep.dev = dev;
402 	hw = &adapter->hw;
403 
404 	/* Core Lock Init*/
405 	IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
406 
407 	/* SYSCTL APIs */
408 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
409 			SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
410 			OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
411 			adapter, 0, ixgbe_set_flowcntl, "I", IXGBE_SYSCTL_DESC_SET_FC);
412 
413         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
414 			SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
415 			OID_AUTO, "enable_aim", CTLFLAG_RW,
416 			&ixgbe_enable_aim, 1, "Interrupt Moderation");
417 
418 	/*
419 	** Allow a kind of speed control by forcing the autoneg
420 	** advertised speed list to only a certain value, this
421 	** supports 1G on 82599 devices, and 100Mb on x540.
422 	*/
423 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
424 			SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
425 			OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
426 			adapter, 0, ixgbe_set_advertise, "I", IXGBE_SYSCTL_DESC_ADV_SPEED);
427 
428 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
429 			SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
430 			OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
431 			0, ixgbe_set_thermal_test, "I", "Thermal Test");
432 
433 	/* Set up the timer callout */
434 	callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
435 
436 	/* Determine hardware revision */
437 	ixgbe_identify_hardware(adapter);
438 
439 	/* Do base PCI setup - map BAR0 */
440 	if (ixgbe_allocate_pci_resources(adapter)) {
441 		device_printf(dev, "Allocation of PCI resources failed\n");
442 		error = ENXIO;
443 		goto err_out;
444 	}
445 
446 	/* Do descriptor calc and sanity checks */
447 	if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
448 	    ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
449 		device_printf(dev, "TXD config issue, using default!\n");
450 		adapter->num_tx_desc = DEFAULT_TXD;
451 	} else
452 		adapter->num_tx_desc = ixgbe_txd;
453 
454 	/*
455 	** With many RX rings it is easy to exceed the
456 	** system mbuf allocation. Tuning nmbclusters
457 	** can alleviate this.
458 	*/
459 	if (nmbclusters > 0) {
460 		int s;
461 		s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
462 		if (s > nmbclusters) {
463 			device_printf(dev, "RX Descriptors exceed "
464 			    "system mbuf max, using default instead!\n");
465 			ixgbe_rxd = DEFAULT_RXD;
466 		}
467 	}
468 
469 	if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
470 	    ixgbe_rxd < MIN_RXD || ixgbe_rxd > MAX_RXD) {
471 		device_printf(dev, "RXD config issue, using default!\n");
472 		adapter->num_rx_desc = DEFAULT_RXD;
473 	} else
474 		adapter->num_rx_desc = ixgbe_rxd;
475 
476 	/* Allocate our TX/RX Queues */
477 	if (ixgbe_allocate_queues(adapter)) {
478 		error = ENOMEM;
479 		goto err_out;
480 	}
481 
482 	/* Allocate multicast array memory. */
483 	adapter->mta = malloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
484 	    MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
485 	if (adapter->mta == NULL) {
486 		device_printf(dev, "Can not allocate multicast setup array\n");
487 		error = ENOMEM;
488 		goto err_late;
489 	}
490 
491 	/* Initialize the shared code */
492 	hw->allow_unsupported_sfp = allow_unsupported_sfp;
493 	error = ixgbe_init_shared_code(hw);
494 	if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
495 		/*
496 		** No optics in this port, set up
497 		** so the timer routine will probe
498 		** for later insertion.
499 		*/
500 		adapter->sfp_probe = TRUE;
501 		error = 0;
502 	} else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
503 		device_printf(dev,"Unsupported SFP+ module detected!\n");
504 		error = EIO;
505 		goto err_late;
506 	} else if (error) {
507 		device_printf(dev,"Unable to initialize the shared code\n");
508 		error = EIO;
509 		goto err_late;
510 	}
511 
512 	/* Make sure we have a good EEPROM before we read from it */
513 	if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
514 		device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
515 		error = EIO;
516 		goto err_late;
517 	}
518 
519 	error = ixgbe_init_hw(hw);
520 	switch (error) {
521 	case IXGBE_ERR_EEPROM_VERSION:
522 		device_printf(dev, "This device is a pre-production adapter/"
523 		    "LOM.  Please be aware there may be issues associated "
524 		    "with your hardware.\n If you are experiencing problems "
525 		    "please contact your Intel or hardware representative "
526 		    "who provided you with this hardware.\n");
527 		break;
528 	case IXGBE_ERR_SFP_NOT_SUPPORTED:
529 		device_printf(dev,"Unsupported SFP+ Module\n");
530 		error = EIO;
531 		goto err_late;
532 	case IXGBE_ERR_SFP_NOT_PRESENT:
533 		device_printf(dev,"No SFP+ Module found\n");
534 		/* falls thru */
535 	default:
536 		break;
537 	}
538 
539 	/* Detect and set physical type */
540 	ixgbe_setup_optics(adapter);
541 
542 	if ((adapter->msix > 1) && (ixgbe_enable_msix))
543 		error = ixgbe_allocate_msix(adapter);
544 	else
545 		error = ixgbe_allocate_legacy(adapter);
546 	if (error)
547 		goto err_late;
548 
549 	/* Setup OS specific network interface */
550 	if (ixgbe_setup_interface(dev, adapter) != 0)
551 		goto err_late;
552 
553 	/* Initialize statistics */
554 	ixgbe_update_stats_counters(adapter);
555 
556 	/* Register for VLAN events */
557 	adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
558 	    ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
559 	adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
560 	    ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
561 
562         /*
563 	** Check PCIE slot type/speed/width
564 	*/
565 	ixgbe_get_slot_info(hw);
566 
567 
568 	/* Set an initial default flow control value */
569 	adapter->fc = ixgbe_fc_full;
570 
571 	/* let hardware know driver is loaded */
572 	ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
573 	ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
574 	IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
575 
576 	ixgbe_add_hw_stats(adapter);
577 
578 #ifdef DEV_NETMAP
579 	ixgbe_netmap_attach(adapter);
580 #endif /* DEV_NETMAP */
581 	INIT_DEBUGOUT("ixgbe_attach: end");
582 	return (0);
583 
584 err_late:
585 	ixgbe_free_transmit_structures(adapter);
586 	ixgbe_free_receive_structures(adapter);
587 err_out:
588 	if (adapter->ifp != NULL)
589 		if_free(adapter->ifp);
590 	ixgbe_free_pci_resources(adapter);
591 	free(adapter->mta, M_DEVBUF);
592 	return (error);
593 }
594 
595 /*********************************************************************
596  *  Device removal routine
597  *
598  *  The detach entry point is called when the driver is being removed.
599  *  This routine stops the adapter and deallocates all the resources
600  *  that were allocated for driver operation.
601  *
602  *  return 0 on success, positive on failure
603  *********************************************************************/
604 
605 static int
606 ixgbe_detach(device_t dev)
607 {
608 	struct adapter *adapter = device_get_softc(dev);
609 	struct ix_queue *que = adapter->queues;
610 	struct tx_ring *txr = adapter->tx_rings;
611 	u32	ctrl_ext;
612 
613 	INIT_DEBUGOUT("ixgbe_detach: begin");
614 
615 	/* Make sure VLANS are not using driver */
616 	if (adapter->ifp->if_vlantrunk != NULL) {
617 		device_printf(dev,"Vlan in use, detach first\n");
618 		return (EBUSY);
619 	}
620 
621 	IXGBE_CORE_LOCK(adapter);
622 	ixgbe_stop(adapter);
623 	IXGBE_CORE_UNLOCK(adapter);
624 
625 	for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
626 		if (que->tq) {
627 #ifndef IXGBE_LEGACY_TX
628 			taskqueue_drain(que->tq, &txr->txq_task);
629 #endif
630 			taskqueue_drain(que->tq, &que->que_task);
631 			taskqueue_free(que->tq);
632 		}
633 	}
634 
635 	/* Drain the Link queue */
636 	if (adapter->tq) {
637 		taskqueue_drain(adapter->tq, &adapter->link_task);
638 		taskqueue_drain(adapter->tq, &adapter->mod_task);
639 		taskqueue_drain(adapter->tq, &adapter->msf_task);
640 #ifdef IXGBE_FDIR
641 		taskqueue_drain(adapter->tq, &adapter->fdir_task);
642 #endif
643 		taskqueue_free(adapter->tq);
644 	}
645 
646 	/* let hardware know driver is unloading */
647 	ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
648 	ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
649 	IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
650 
651 	/* Unregister VLAN events */
652 	if (adapter->vlan_attach != NULL)
653 		EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
654 	if (adapter->vlan_detach != NULL)
655 		EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
656 
657 	ether_ifdetach(adapter->ifp);
658 	callout_drain(&adapter->timer);
659 #ifdef DEV_NETMAP
660 	netmap_detach(adapter->ifp);
661 #endif /* DEV_NETMAP */
662 	ixgbe_free_pci_resources(adapter);
663 	bus_generic_detach(dev);
664 	if_free(adapter->ifp);
665 
666 	ixgbe_free_transmit_structures(adapter);
667 	ixgbe_free_receive_structures(adapter);
668 	free(adapter->mta, M_DEVBUF);
669 
670 	IXGBE_CORE_LOCK_DESTROY(adapter);
671 	return (0);
672 }
673 
674 /*********************************************************************
675  *
676  *  Shutdown entry point
677  *
678  **********************************************************************/
679 
680 static int
681 ixgbe_shutdown(device_t dev)
682 {
683 	struct adapter *adapter = device_get_softc(dev);
684 	IXGBE_CORE_LOCK(adapter);
685 	ixgbe_stop(adapter);
686 	IXGBE_CORE_UNLOCK(adapter);
687 	return (0);
688 }
689 
690 
691 /*********************************************************************
692  *  Ioctl entry point
693  *
694  *  ixgbe_ioctl is called when the user wants to configure the
695  *  interface.
696  *
697  *  return 0 on success, positive on failure
698  **********************************************************************/
699 
700 static int
701 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
702 {
703 	struct adapter	*adapter = ifp->if_softc;
704 	struct ifreq	*ifr = (struct ifreq *) data;
705 #if defined(INET) || defined(INET6)
706 	struct ifaddr *ifa = (struct ifaddr *)data;
707 	bool		avoid_reset = FALSE;
708 #endif
709 	int             error = 0;
710 
711 	switch (command) {
712 
713         case SIOCSIFADDR:
714 #ifdef INET
715 		if (ifa->ifa_addr->sa_family == AF_INET)
716 			avoid_reset = TRUE;
717 #endif
718 #ifdef INET6
719 		if (ifa->ifa_addr->sa_family == AF_INET6)
720 			avoid_reset = TRUE;
721 #endif
722 #if defined(INET) || defined(INET6)
723 		/*
724 		** Calling init results in link renegotiation,
725 		** so we avoid doing it when possible.
726 		*/
727 		if (avoid_reset) {
728 			ifp->if_flags |= IFF_UP;
729 			if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
730 				ixgbe_init(adapter);
731 			if (!(ifp->if_flags & IFF_NOARP))
732 				arp_ifinit(ifp, ifa);
733 		} else
734 			error = ether_ioctl(ifp, command, data);
735 #endif
736 		break;
737 	case SIOCSIFMTU:
738 		IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
739 		if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
740 			error = EINVAL;
741 		} else {
742 			IXGBE_CORE_LOCK(adapter);
743 			ifp->if_mtu = ifr->ifr_mtu;
744 			adapter->max_frame_size =
745 				ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
746 			ixgbe_init_locked(adapter);
747 			IXGBE_CORE_UNLOCK(adapter);
748 		}
749 		break;
750 	case SIOCSIFFLAGS:
751 		IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
752 		IXGBE_CORE_LOCK(adapter);
753 		if (ifp->if_flags & IFF_UP) {
754 			if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
755 				if ((ifp->if_flags ^ adapter->if_flags) &
756 				    (IFF_PROMISC | IFF_ALLMULTI)) {
757 					ixgbe_set_promisc(adapter);
758                                 }
759 			} else
760 				ixgbe_init_locked(adapter);
761 		} else
762 			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
763 				ixgbe_stop(adapter);
764 		adapter->if_flags = ifp->if_flags;
765 		IXGBE_CORE_UNLOCK(adapter);
766 		break;
767 	case SIOCADDMULTI:
768 	case SIOCDELMULTI:
769 		IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
770 		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
771 			IXGBE_CORE_LOCK(adapter);
772 			ixgbe_disable_intr(adapter);
773 			ixgbe_set_multi(adapter);
774 			ixgbe_enable_intr(adapter);
775 			IXGBE_CORE_UNLOCK(adapter);
776 		}
777 		break;
778 	case SIOCSIFMEDIA:
779 	case SIOCGIFMEDIA:
780 		IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
781 		error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
782 		break;
783 	case SIOCSIFCAP:
784 	{
785 		int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
786 		IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
787 		if (mask & IFCAP_HWCSUM)
788 			ifp->if_capenable ^= IFCAP_HWCSUM;
789 		if (mask & IFCAP_TSO4)
790 			ifp->if_capenable ^= IFCAP_TSO4;
791 		if (mask & IFCAP_TSO6)
792 			ifp->if_capenable ^= IFCAP_TSO6;
793 		if (mask & IFCAP_LRO)
794 			ifp->if_capenable ^= IFCAP_LRO;
795 		if (mask & IFCAP_VLAN_HWTAGGING)
796 			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
797 		if (mask & IFCAP_VLAN_HWFILTER)
798 			ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
799 		if (mask & IFCAP_VLAN_HWTSO)
800 			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
801 		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
802 			IXGBE_CORE_LOCK(adapter);
803 			ixgbe_init_locked(adapter);
804 			IXGBE_CORE_UNLOCK(adapter);
805 		}
806 		VLAN_CAPABILITIES(ifp);
807 		break;
808 	}
809 #if __FreeBSD_version >= 1100036
810 	case SIOCGI2C:
811 	{
812 		struct ixgbe_hw *hw = &adapter->hw;
813 		struct ifi2creq i2c;
814 		int i;
815 		IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)");
816 		error = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
817 		if (error != 0)
818 			break;
819 		if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
820 			error = EINVAL;
821 			break;
822 		}
823 		if (i2c.len > sizeof(i2c.data)) {
824 			error = EINVAL;
825 			break;
826 		}
827 
828 		for (i = 0; i < i2c.len; i++)
829 			hw->phy.ops.read_i2c_byte(hw, i2c.offset + i,
830 			    i2c.dev_addr, &i2c.data[i]);
831 		error = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
832 		break;
833 	}
834 #endif
835 	default:
836 		IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
837 		error = ether_ioctl(ifp, command, data);
838 		break;
839 	}
840 
841 	return (error);
842 }
843 
844 /*********************************************************************
845  *  Init entry point
846  *
847  *  This routine is used in two ways. It is used by the stack as
848  *  init entry point in network interface structure. It is also used
849  *  by the driver as a hw/sw initialization routine to get to a
850  *  consistent state.
851  *
852  *  return 0 on success, positive on failure
853  **********************************************************************/
854 #define IXGBE_MHADD_MFS_SHIFT 16
855 
856 static void
857 ixgbe_init_locked(struct adapter *adapter)
858 {
859 	struct ifnet   *ifp = adapter->ifp;
860 	device_t 	dev = adapter->dev;
861 	struct ixgbe_hw *hw = &adapter->hw;
862 	u32		k, txdctl, mhadd, gpie;
863 	u32		rxdctl, rxctrl;
864 
865 	mtx_assert(&adapter->core_mtx, MA_OWNED);
866 	INIT_DEBUGOUT("ixgbe_init_locked: begin");
867 	hw->adapter_stopped = FALSE;
868 	ixgbe_stop_adapter(hw);
869         callout_stop(&adapter->timer);
870 
871         /* reprogram the RAR[0] in case user changed it. */
872         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
873 
874 	/* Get the latest mac address, User can use a LAA */
875 	bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
876 	      IXGBE_ETH_LENGTH_OF_ADDRESS);
877 	ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
878 	hw->addr_ctrl.rar_used_count = 1;
879 
880 	/* Set the various hardware offload abilities */
881 	ifp->if_hwassist = 0;
882 	if (ifp->if_capenable & IFCAP_TSO)
883 		ifp->if_hwassist |= CSUM_TSO;
884 	if (ifp->if_capenable & IFCAP_TXCSUM) {
885 		ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
886 #if __FreeBSD_version >= 800000
887 		if (hw->mac.type != ixgbe_mac_82598EB)
888 			ifp->if_hwassist |= CSUM_SCTP;
889 #endif
890 	}
891 
892 	/* Prepare transmit descriptors and buffers */
893 	if (ixgbe_setup_transmit_structures(adapter)) {
894 		device_printf(dev,"Could not setup transmit structures\n");
895 		ixgbe_stop(adapter);
896 		return;
897 	}
898 
899 	ixgbe_init_hw(hw);
900 	ixgbe_initialize_transmit_units(adapter);
901 
902 	/* Setup Multicast table */
903 	ixgbe_set_multi(adapter);
904 
905 	/*
906 	** Determine the correct mbuf pool
907 	** for doing jumbo frames
908 	*/
909 	if (adapter->max_frame_size <= 2048)
910 		adapter->rx_mbuf_sz = MCLBYTES;
911 	else if (adapter->max_frame_size <= 4096)
912 		adapter->rx_mbuf_sz = MJUMPAGESIZE;
913 	else if (adapter->max_frame_size <= 9216)
914 		adapter->rx_mbuf_sz = MJUM9BYTES;
915 	else
916 		adapter->rx_mbuf_sz = MJUM16BYTES;
917 
918 	/* Prepare receive descriptors and buffers */
919 	if (ixgbe_setup_receive_structures(adapter)) {
920 		device_printf(dev,"Could not setup receive structures\n");
921 		ixgbe_stop(adapter);
922 		return;
923 	}
924 
925 	/* Configure RX settings */
926 	ixgbe_initialize_receive_units(adapter);
927 
928 	gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
929 
930 	/* Enable Fan Failure Interrupt */
931 	gpie |= IXGBE_SDP1_GPIEN_BY_MAC(hw);
932 
933 	/* Add for Module detection */
934 	if (hw->mac.type == ixgbe_mac_82599EB)
935 		gpie |= IXGBE_SDP2_GPIEN_BY_MAC(hw);
936 
937 	/* Thermal Failure Detection */
938 	if (hw->mac.type == ixgbe_mac_X540)
939 		gpie |= IXGBE_SDP0_GPIEN_BY_MAC(hw);
940 
941 	if (adapter->msix > 1) {
942 		/* Enable Enhanced MSIX mode */
943 		gpie |= IXGBE_GPIE_MSIX_MODE;
944 		gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
945 		    IXGBE_GPIE_OCD;
946 	}
947 	IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
948 
949 	/* Set MTU size */
950 	if (ifp->if_mtu > ETHERMTU) {
951 		mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
952 		mhadd &= ~IXGBE_MHADD_MFS_MASK;
953 		mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
954 		IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
955 	}
956 
957 	/* Now enable all the queues */
958 
959 	for (int i = 0; i < adapter->num_queues; i++) {
960 		txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
961 		txdctl |= IXGBE_TXDCTL_ENABLE;
962 		/* Set WTHRESH to 8, burst writeback */
963 		txdctl |= (8 << 16);
964 		/*
965 		 * When the internal queue falls below PTHRESH (32),
966 		 * start prefetching as long as there are at least
967 		 * HTHRESH (1) buffers ready. The values are taken
968 		 * from the Intel linux driver 3.8.21.
969 		 * Prefetching enables tx line rate even with 1 queue.
970 		 */
971 		txdctl |= (32 << 0) | (1 << 8);
972 		IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
973 	}
974 
975 	for (int i = 0; i < adapter->num_queues; i++) {
976 		rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
977 		if (hw->mac.type == ixgbe_mac_82598EB) {
978 			/*
979 			** PTHRESH = 21
980 			** HTHRESH = 4
981 			** WTHRESH = 8
982 			*/
983 			rxdctl &= ~0x3FFFFF;
984 			rxdctl |= 0x080420;
985 		}
986 		rxdctl |= IXGBE_RXDCTL_ENABLE;
987 		IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
988 		for (k = 0; k < 10; k++) {
989 			if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
990 			    IXGBE_RXDCTL_ENABLE)
991 				break;
992 			else
993 				msec_delay(1);
994 		}
995 		wmb();
996 #ifdef DEV_NETMAP
997 		/*
998 		 * In netmap mode, we must preserve the buffers made
999 		 * available to userspace before the if_init()
1000 		 * (this is true by default on the TX side, because
1001 		 * init makes all buffers available to userspace).
1002 		 *
1003 		 * netmap_reset() and the device specific routines
1004 		 * (e.g. ixgbe_setup_receive_rings()) map these
1005 		 * buffers at the end of the NIC ring, so here we
1006 		 * must set the RDT (tail) register to make sure
1007 		 * they are not overwritten.
1008 		 *
1009 		 * In this driver the NIC ring starts at RDH = 0,
1010 		 * RDT points to the last slot available for reception (?),
1011 		 * so RDT = num_rx_desc - 1 means the whole ring is available.
1012 		 */
1013 		if (ifp->if_capenable & IFCAP_NETMAP) {
1014 			struct netmap_adapter *na = NA(adapter->ifp);
1015 			struct netmap_kring *kring = &na->rx_rings[i];
1016 			int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
1017 
1018 			IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1019 		} else
1020 #endif /* DEV_NETMAP */
1021 		IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1022 	}
1023 
1024 	/* Enable Receive engine */
1025 	rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1026 	if (hw->mac.type == ixgbe_mac_82598EB)
1027 		rxctrl |= IXGBE_RXCTRL_DMBYPS;
1028 	rxctrl |= IXGBE_RXCTRL_RXEN;
1029 	ixgbe_enable_rx_dma(hw, rxctrl);
1030 
1031 	callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1032 
1033 	/* Set up MSI/X routing */
1034 	if (ixgbe_enable_msix)  {
1035 		ixgbe_configure_ivars(adapter);
1036 		/* Set up auto-mask */
1037 		if (hw->mac.type == ixgbe_mac_82598EB)
1038 			IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1039 		else {
1040 			IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1041 			IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1042 		}
1043 	} else {  /* Simple settings for Legacy/MSI */
1044                 ixgbe_set_ivar(adapter, 0, 0, 0);
1045                 ixgbe_set_ivar(adapter, 0, 0, 1);
1046 		IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1047 	}
1048 
1049 #ifdef IXGBE_FDIR
1050 	/* Init Flow director */
1051 	if (hw->mac.type != ixgbe_mac_82598EB) {
1052 		u32 hdrm = 32 << fdir_pballoc;
1053 
1054 		hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1055 		ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1056 	}
1057 #endif
1058 
1059 	/*
1060 	** Check on any SFP devices that
1061 	** need to be kick-started
1062 	*/
1063 	if (hw->phy.type == ixgbe_phy_none) {
1064 		int err = hw->phy.ops.identify(hw);
1065 		if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1066                 	device_printf(dev,
1067 			    "Unsupported SFP+ module type was detected.\n");
1068 			return;
1069         	}
1070 	}
1071 
1072 	/* Set moderation on the Link interrupt */
1073 	IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->vector), IXGBE_LINK_ITR);
1074 
1075 	/* Config/Enable Link */
1076 	ixgbe_config_link(adapter);
1077 
1078 	/* Hardware Packet Buffer & Flow Control setup */
1079 	{
1080 		u32 rxpb, frame, size, tmp;
1081 
1082 		frame = adapter->max_frame_size;
1083 
1084 		/* Calculate High Water */
1085 		switch (hw->mac.type) {
1086 		case ixgbe_mac_X540:
1087 		case ixgbe_mac_X550:
1088 		case ixgbe_mac_X550EM_a:
1089 		case ixgbe_mac_X550EM_x:
1090 			tmp = IXGBE_DV_X540(frame, frame);
1091 			break;
1092 		default:
1093 			tmp = IXGBE_DV(frame, frame);
1094 			break;
1095 		}
1096 		size = IXGBE_BT2KB(tmp);
1097 		rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1098 		hw->fc.high_water[0] = rxpb - size;
1099 
1100 		/* Now calculate Low Water */
1101 		switch (hw->mac.type) {
1102 		case ixgbe_mac_X540:
1103 		case ixgbe_mac_X550:
1104 		case ixgbe_mac_X550EM_a:
1105 		case ixgbe_mac_X550EM_x:
1106 			tmp = IXGBE_LOW_DV_X540(frame);
1107 			break;
1108 		default:
1109 			tmp = IXGBE_LOW_DV(frame);
1110 			break;
1111 		}
1112 		hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1113 
1114 		hw->fc.requested_mode = adapter->fc;
1115 		hw->fc.pause_time = IXGBE_FC_PAUSE;
1116 		hw->fc.send_xon = TRUE;
1117 	}
1118 	/* Initialize the FC settings */
1119 	ixgbe_start_hw(hw);
1120 
1121 	/* Set up VLAN support and filter */
1122 	ixgbe_setup_vlan_hw_support(adapter);
1123 
1124 	/* And now turn on interrupts */
1125 	ixgbe_enable_intr(adapter);
1126 
1127 	/* Now inform the stack we're ready */
1128 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
1129 
1130 	return;
1131 }
1132 
1133 static void
1134 ixgbe_init(void *arg)
1135 {
1136 	struct adapter *adapter = arg;
1137 
1138 	IXGBE_CORE_LOCK(adapter);
1139 	ixgbe_init_locked(adapter);
1140 	IXGBE_CORE_UNLOCK(adapter);
1141 	return;
1142 }
1143 
1144 
1145 /*
1146 **
1147 ** MSIX Interrupt Handlers and Tasklets
1148 **
1149 */
1150 
1151 static inline void
1152 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1153 {
1154 	struct ixgbe_hw *hw = &adapter->hw;
1155 	u64	queue = (u64)(1 << vector);
1156 	u32	mask;
1157 
1158 	if (hw->mac.type == ixgbe_mac_82598EB) {
1159                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1160                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1161 	} else {
1162                 mask = (queue & 0xFFFFFFFF);
1163                 if (mask)
1164                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1165                 mask = (queue >> 32);
1166                 if (mask)
1167                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1168 	}
1169 }
1170 
1171 static inline void
1172 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1173 {
1174 	struct ixgbe_hw *hw = &adapter->hw;
1175 	u64	queue = (u64)(1 << vector);
1176 	u32	mask;
1177 
1178 	if (hw->mac.type == ixgbe_mac_82598EB) {
1179                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1180                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1181 	} else {
1182                 mask = (queue & 0xFFFFFFFF);
1183                 if (mask)
1184                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1185                 mask = (queue >> 32);
1186                 if (mask)
1187                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1188 	}
1189 }
1190 
1191 static void
1192 ixgbe_handle_que(void *context, int pending)
1193 {
1194 	struct ix_queue *que = context;
1195 	struct adapter  *adapter = que->adapter;
1196 	struct tx_ring  *txr = que->txr;
1197 	struct ifnet    *ifp = adapter->ifp;
1198 
1199 	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1200 		ixgbe_rxeof(que);
1201 		IXGBE_TX_LOCK(txr);
1202 		ixgbe_txeof(txr);
1203 #ifndef IXGBE_LEGACY_TX
1204 		if (!drbr_empty(ifp, txr->br))
1205 			ixgbe_mq_start_locked(ifp, txr);
1206 #else
1207 		if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1208 			ixgbe_start_locked(txr, ifp);
1209 #endif
1210 		IXGBE_TX_UNLOCK(txr);
1211 	}
1212 
1213 	/* Reenable this interrupt */
1214 	if (que->res != NULL)
1215 		ixgbe_enable_queue(adapter, que->msix);
1216 	else
1217 		ixgbe_enable_intr(adapter);
1218 	return;
1219 }
1220 
1221 
1222 /*********************************************************************
1223  *
1224  *  Legacy Interrupt Service routine
1225  *
1226  **********************************************************************/
1227 
1228 static void
1229 ixgbe_legacy_irq(void *arg)
1230 {
1231 	struct ix_queue *que = arg;
1232 	struct adapter	*adapter = que->adapter;
1233 	struct ixgbe_hw	*hw = &adapter->hw;
1234 	struct ifnet    *ifp = adapter->ifp;
1235 	struct 		tx_ring *txr = adapter->tx_rings;
1236 	bool		more;
1237 	u32       	reg_eicr;
1238 
1239 
1240 	reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1241 
1242 	++que->irqs;
1243 	if (reg_eicr == 0) {
1244 		ixgbe_enable_intr(adapter);
1245 		return;
1246 	}
1247 
1248 	more = ixgbe_rxeof(que);
1249 
1250 	IXGBE_TX_LOCK(txr);
1251 	ixgbe_txeof(txr);
1252 #ifdef IXGBE_LEGACY_TX
1253 	if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1254 		ixgbe_start_locked(txr, ifp);
1255 #else
1256 	if (!drbr_empty(ifp, txr->br))
1257 		ixgbe_mq_start_locked(ifp, txr);
1258 #endif
1259 	IXGBE_TX_UNLOCK(txr);
1260 
1261 	/* Check for fan failure */
1262 	if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1263 	    (reg_eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) {
1264                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1265 		    "REPLACE IMMEDIATELY!!\n");
1266 		IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
1267 	}
1268 
1269 	/* Link status change */
1270 	if (reg_eicr & IXGBE_EICR_LSC)
1271 		taskqueue_enqueue(adapter->tq, &adapter->link_task);
1272 
1273 	if (more)
1274 		taskqueue_enqueue(que->tq, &que->que_task);
1275 	else
1276 		ixgbe_enable_intr(adapter);
1277 	return;
1278 }
1279 
1280 
1281 /*********************************************************************
1282  *
1283  *  MSIX Queue Interrupt Service routine
1284  *
1285  **********************************************************************/
1286 void
1287 ixgbe_msix_que(void *arg)
1288 {
1289 	struct ix_queue	*que = arg;
1290 	struct adapter  *adapter = que->adapter;
1291 	struct ifnet    *ifp = adapter->ifp;
1292 	struct tx_ring	*txr = que->txr;
1293 	struct rx_ring	*rxr = que->rxr;
1294 	bool		more;
1295 	u32		newitr = 0;
1296 
1297 	/* Protect against spurious interrupts */
1298 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1299 		return;
1300 
1301 	ixgbe_disable_queue(adapter, que->msix);
1302 	++que->irqs;
1303 
1304 	more = ixgbe_rxeof(que);
1305 
1306 	IXGBE_TX_LOCK(txr);
1307 	ixgbe_txeof(txr);
1308 #ifdef IXGBE_LEGACY_TX
1309 	if (!IFQ_DRV_IS_EMPTY(ifp->if_snd))
1310 		ixgbe_start_locked(txr, ifp);
1311 #else
1312 	if (!drbr_empty(ifp, txr->br))
1313 		ixgbe_mq_start_locked(ifp, txr);
1314 #endif
1315 	IXGBE_TX_UNLOCK(txr);
1316 
1317 	/* Do AIM now? */
1318 
1319 	if (ixgbe_enable_aim == FALSE)
1320 		goto no_calc;
1321 	/*
1322 	** Do Adaptive Interrupt Moderation:
1323         **  - Write out last calculated setting
1324 	**  - Calculate based on average size over
1325 	**    the last interval.
1326 	*/
1327         if (que->eitr_setting)
1328                 IXGBE_WRITE_REG(&adapter->hw,
1329                     IXGBE_EITR(que->msix), que->eitr_setting);
1330 
1331         que->eitr_setting = 0;
1332 
1333         /* Idle, do nothing */
1334         if ((txr->bytes == 0) && (rxr->bytes == 0))
1335                 goto no_calc;
1336 
1337 	if ((txr->bytes) && (txr->packets))
1338                	newitr = txr->bytes/txr->packets;
1339 	if ((rxr->bytes) && (rxr->packets))
1340 		newitr = max(newitr,
1341 		    (rxr->bytes / rxr->packets));
1342 	newitr += 24; /* account for hardware frame, crc */
1343 
1344 	/* set an upper boundary */
1345 	newitr = min(newitr, 3000);
1346 
1347 	/* Be nice to the mid range */
1348 	if ((newitr > 300) && (newitr < 1200))
1349 		newitr = (newitr / 3);
1350 	else
1351 		newitr = (newitr / 2);
1352 
1353         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1354                 newitr |= newitr << 16;
1355         else
1356                 newitr |= IXGBE_EITR_CNT_WDIS;
1357 
1358         /* save for next interrupt */
1359         que->eitr_setting = newitr;
1360 
1361         /* Reset state */
1362         txr->bytes = 0;
1363         txr->packets = 0;
1364         rxr->bytes = 0;
1365         rxr->packets = 0;
1366 
1367 no_calc:
1368 	if (more)
1369 		taskqueue_enqueue(que->tq, &que->que_task);
1370 	else
1371 		ixgbe_enable_queue(adapter, que->msix);
1372 	return;
1373 }
1374 
1375 
1376 static void
1377 ixgbe_msix_link(void *arg)
1378 {
1379 	struct adapter	*adapter = arg;
1380 	struct ixgbe_hw *hw = &adapter->hw;
1381 	u32		reg_eicr;
1382 
1383 	++adapter->vector_irq;
1384 
1385 	/* First get the cause */
1386 	reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1387 	/* Be sure the queue bits are not cleared */
1388 	reg_eicr &= ~IXGBE_EICR_RTX_QUEUE;
1389 	/* Clear interrupt with write */
1390 	IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1391 
1392 	/* Link status change */
1393 	if (reg_eicr & IXGBE_EICR_LSC)
1394 		taskqueue_enqueue(adapter->tq, &adapter->link_task);
1395 
1396 	if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1397 #ifdef IXGBE_FDIR
1398 		if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1399 			/* This is probably overkill :) */
1400 			if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1401 				return;
1402                 	/* Disable the interrupt */
1403 			IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1404 			taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1405 		} else
1406 #endif
1407 		if (reg_eicr & IXGBE_EICR_ECC) {
1408                 	device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1409 			    "Please Reboot!!\n");
1410 			IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1411 		} else
1412 
1413 		if (ixgbe_is_sfp(hw)) {
1414 			if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1415 				IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
1416 				taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1417 			} else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1418 				IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2_BY_MAC(hw));
1419 				taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1420 			}
1421 		}
1422         }
1423 
1424 	/* Check for fan failure */
1425 	if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1426 	    (reg_eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) {
1427                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1428 		    "REPLACE IMMEDIATELY!!\n");
1429 		IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
1430 	}
1431 
1432 	/* Check for over temp condition */
1433 	switch (hw->mac.type) {
1434 	case ixgbe_mac_X540:
1435 	case ixgbe_mac_X550:
1436 	case ixgbe_mac_X550EM_a:
1437 		if (reg_eicr & IXGBE_EICR_TS) {
1438 			device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1439 			    "PHY IS SHUT DOWN!!\n");
1440 			device_printf(adapter->dev, "System shutdown required\n");
1441 			IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS);
1442 		}
1443 		break;
1444 	default:
1445 		/* Other MACs have no thermal sensor interrupt */
1446 		break;
1447 	}
1448 
1449 	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1450 	return;
1451 }
1452 
1453 /*********************************************************************
1454  *
1455  *  Media Ioctl callback
1456  *
1457  *  This routine is called whenever the user queries the status of
1458  *  the interface using ifconfig.
1459  *
1460  **********************************************************************/
1461 static void
1462 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1463 {
1464 	struct adapter *adapter = ifp->if_softc;
1465 	struct ixgbe_hw *hw = &adapter->hw;
1466 	int layer;
1467 
1468 	INIT_DEBUGOUT("ixgbe_media_status: begin");
1469 	IXGBE_CORE_LOCK(adapter);
1470 	ixgbe_update_link_status(adapter);
1471 
1472 	ifmr->ifm_status = IFM_AVALID;
1473 	ifmr->ifm_active = IFM_ETHER;
1474 
1475 	if (!adapter->link_active) {
1476 		IXGBE_CORE_UNLOCK(adapter);
1477 		return;
1478 	}
1479 
1480 	ifmr->ifm_status |= IFM_ACTIVE;
1481 	layer = ixgbe_get_supported_physical_layer(hw);
1482 
1483 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T ||
1484 	    layer & IXGBE_PHYSICAL_LAYER_1000BASE_T ||
1485 	    layer & IXGBE_PHYSICAL_LAYER_100BASE_TX)
1486 		switch (adapter->link_speed) {
1487 		case IXGBE_LINK_SPEED_10GB_FULL:
1488 			ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
1489 			break;
1490 		case IXGBE_LINK_SPEED_1GB_FULL:
1491 			ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1492 			break;
1493 		case IXGBE_LINK_SPEED_100_FULL:
1494 			ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1495 			break;
1496 		}
1497 	if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU ||
1498 	    layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA)
1499 		switch (adapter->link_speed) {
1500 		case IXGBE_LINK_SPEED_10GB_FULL:
1501 			ifmr->ifm_active |= IFM_10G_TWINAX | IFM_FDX;
1502 			break;
1503 		}
1504 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR)
1505 		switch (adapter->link_speed) {
1506 		case IXGBE_LINK_SPEED_10GB_FULL:
1507 			ifmr->ifm_active |= IFM_10G_LR | IFM_FDX;
1508 			break;
1509 		case IXGBE_LINK_SPEED_1GB_FULL:
1510 			ifmr->ifm_active |= IFM_1000_LX | IFM_FDX;
1511 			break;
1512 		}
1513 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LRM)
1514 		switch (adapter->link_speed) {
1515 		case IXGBE_LINK_SPEED_10GB_FULL:
1516 			ifmr->ifm_active |= IFM_10G_LRM | IFM_FDX;
1517 			break;
1518 		case IXGBE_LINK_SPEED_1GB_FULL:
1519 			ifmr->ifm_active |= IFM_1000_LX | IFM_FDX;
1520 			break;
1521 		}
1522 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR ||
1523 	    layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX)
1524 		switch (adapter->link_speed) {
1525 		case IXGBE_LINK_SPEED_10GB_FULL:
1526 			ifmr->ifm_active |= IFM_10G_SR | IFM_FDX;
1527 			break;
1528 		case IXGBE_LINK_SPEED_1GB_FULL:
1529 			ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
1530 			break;
1531 		}
1532 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4)
1533 		switch (adapter->link_speed) {
1534 		case IXGBE_LINK_SPEED_10GB_FULL:
1535 			ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX;
1536 			break;
1537 		}
1538 	/*
1539 	** XXX: These need to use the proper media types once
1540 	** they're added.
1541 	*/
1542 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR)
1543 		switch (adapter->link_speed) {
1544 		case IXGBE_LINK_SPEED_10GB_FULL:
1545 			ifmr->ifm_active |= IFM_10_T | IFM_FDX;
1546 			break;
1547 		case IXGBE_LINK_SPEED_1GB_FULL:
1548 			ifmr->ifm_active |= IFM_10_5 | IFM_FDX;
1549 			break;
1550 		}
1551 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4
1552 	    || layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX)
1553 		switch (adapter->link_speed) {
1554 		case IXGBE_LINK_SPEED_10GB_FULL:
1555 			ifmr->ifm_active |= IFM_10_2 | IFM_FDX;
1556 			break;
1557 		case IXGBE_LINK_SPEED_1GB_FULL:
1558 			ifmr->ifm_active |= IFM_10_5 | IFM_FDX;
1559 			break;
1560 		}
1561 
1562 	/* If nothing is recognized... */
1563 	if (IFM_SUBTYPE(ifmr->ifm_active) == 0)
1564 		ifmr->ifm_active |= IFM_UNKNOWN;
1565 
1566 #if __FreeBSD_version >= 900025
1567 	/* Flow control setting */
1568 	if (adapter->fc == ixgbe_fc_rx_pause || adapter->fc == ixgbe_fc_full)
1569 		ifmr->ifm_active |= IFM_ETH_RXPAUSE;
1570 	if (adapter->fc == ixgbe_fc_tx_pause || adapter->fc == ixgbe_fc_full)
1571 		ifmr->ifm_active |= IFM_ETH_TXPAUSE;
1572 #endif
1573 
1574 	IXGBE_CORE_UNLOCK(adapter);
1575 
1576 	return;
1577 }
1578 
1579 /*********************************************************************
1580  *
1581  *  Media Ioctl callback
1582  *
1583  *  This routine is called when the user changes speed/duplex using
1584  *  media/mediopt option with ifconfig.
1585  *
1586  **********************************************************************/
1587 static int
1588 ixgbe_media_change(struct ifnet * ifp)
1589 {
1590 	struct adapter *adapter = ifp->if_softc;
1591 	struct ifmedia *ifm = &adapter->media;
1592 	struct ixgbe_hw *hw = &adapter->hw;
1593 	ixgbe_link_speed speed = 0;
1594 
1595 	INIT_DEBUGOUT("ixgbe_media_change: begin");
1596 
1597 	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1598 		return (EINVAL);
1599 
1600 	/*
1601 	** We don't actually need to check against the supported
1602 	** media types of the adapter; ifmedia will take care of
1603 	** that for us.
1604 	** 	NOTE: this relies on falling thru the switch
1605 	**	to get all the values set, it can be confusing.
1606 	*/
1607 	switch (IFM_SUBTYPE(ifm->ifm_media)) {
1608 		case IFM_AUTO:
1609 		case IFM_10G_T:
1610 			speed |= IXGBE_LINK_SPEED_100_FULL;
1611 		case IFM_10G_LRM:
1612 		case IFM_10G_SR:  /* KR, too */
1613 		case IFM_10G_LR:
1614 		case IFM_10G_CX4: /* KX4 for now */
1615 			speed |= IXGBE_LINK_SPEED_1GB_FULL;
1616 		case IFM_10G_TWINAX:
1617 			speed |= IXGBE_LINK_SPEED_10GB_FULL;
1618 			break;
1619 		case IFM_1000_T:
1620 			speed |= IXGBE_LINK_SPEED_100_FULL;
1621 		case IFM_1000_LX:
1622 		case IFM_1000_SX:
1623 		case IFM_1000_CX: /* KX until there's real support */
1624 			speed |= IXGBE_LINK_SPEED_1GB_FULL;
1625 			break;
1626 		case IFM_100_TX:
1627 			speed |= IXGBE_LINK_SPEED_100_FULL;
1628 			break;
1629 		default:
1630 			goto invalid;
1631 	}
1632 
1633 	hw->mac.autotry_restart = TRUE;
1634 	hw->mac.ops.setup_link(hw, speed, TRUE);
1635 	adapter->advertise =
1636 		((speed & IXGBE_LINK_SPEED_10GB_FULL) << 2) |
1637 		((speed & IXGBE_LINK_SPEED_1GB_FULL) << 1) |
1638 		((speed & IXGBE_LINK_SPEED_100_FULL) << 0);
1639 
1640 	return (0);
1641 
1642 invalid:
1643 	device_printf(adapter->dev, "Invalid media type\n");
1644 	return (EINVAL);
1645 }
1646 
1647 static void
1648 ixgbe_set_promisc(struct adapter *adapter)
1649 {
1650 	u_int32_t       reg_rctl;
1651 	struct ifnet   *ifp = adapter->ifp;
1652 	int		mcnt = 0;
1653 
1654 	reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1655 	reg_rctl &= (~IXGBE_FCTRL_UPE);
1656 	if (ifp->if_flags & IFF_ALLMULTI)
1657 		mcnt = MAX_NUM_MULTICAST_ADDRESSES;
1658 	else {
1659 		struct	ifmultiaddr *ifma;
1660 #if __FreeBSD_version < 800000
1661 		IF_ADDR_LOCK(ifp);
1662 #else
1663 		if_maddr_rlock(ifp);
1664 #endif
1665 		TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1666 			if (ifma->ifma_addr->sa_family != AF_LINK)
1667 				continue;
1668 			if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1669 				break;
1670 			mcnt++;
1671 		}
1672 #if __FreeBSD_version < 800000
1673 		IF_ADDR_UNLOCK(ifp);
1674 #else
1675 		if_maddr_runlock(ifp);
1676 #endif
1677 	}
1678 	if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
1679 		reg_rctl &= (~IXGBE_FCTRL_MPE);
1680 	IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1681 
1682 	if (ifp->if_flags & IFF_PROMISC) {
1683 		reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1684 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1685 	} else if (ifp->if_flags & IFF_ALLMULTI) {
1686 		reg_rctl |= IXGBE_FCTRL_MPE;
1687 		reg_rctl &= ~IXGBE_FCTRL_UPE;
1688 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1689 	}
1690 	return;
1691 }
1692 
1693 
1694 /*********************************************************************
1695  *  Multicast Update
1696  *
1697  *  This routine is called whenever multicast address list is updated.
1698  *
1699  **********************************************************************/
1700 #define IXGBE_RAR_ENTRIES 16
1701 
1702 static void
1703 ixgbe_set_multi(struct adapter *adapter)
1704 {
1705 	u32	fctrl;
1706 	u8	*mta;
1707 	u8	*update_ptr;
1708 	struct	ifmultiaddr *ifma;
1709 	int	mcnt = 0;
1710 	struct ifnet   *ifp = adapter->ifp;
1711 
1712 	IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1713 
1714 	mta = adapter->mta;
1715 	bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1716 	    MAX_NUM_MULTICAST_ADDRESSES);
1717 
1718 #if __FreeBSD_version < 800000
1719 	IF_ADDR_LOCK(ifp);
1720 #else
1721 	if_maddr_rlock(ifp);
1722 #endif
1723 	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1724 		if (ifma->ifma_addr->sa_family != AF_LINK)
1725 			continue;
1726 		if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1727 			break;
1728 		bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1729 		    &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1730 		    IXGBE_ETH_LENGTH_OF_ADDRESS);
1731 		mcnt++;
1732 	}
1733 #if __FreeBSD_version < 800000
1734 	IF_ADDR_UNLOCK(ifp);
1735 #else
1736 	if_maddr_runlock(ifp);
1737 #endif
1738 
1739 	fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1740 	fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1741 	if (ifp->if_flags & IFF_PROMISC)
1742 		fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1743 	else if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES ||
1744 	    ifp->if_flags & IFF_ALLMULTI) {
1745 		fctrl |= IXGBE_FCTRL_MPE;
1746 		fctrl &= ~IXGBE_FCTRL_UPE;
1747 	} else
1748 		fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1749 
1750 	IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1751 
1752 	if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) {
1753 		update_ptr = mta;
1754 		ixgbe_update_mc_addr_list(&adapter->hw,
1755 		    update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1756 	}
1757 
1758 	return;
1759 }
1760 
1761 /*
1762  * This is an iterator function now needed by the multicast
1763  * shared code. It simply feeds the shared code routine the
1764  * addresses in the array of ixgbe_set_multi() one by one.
1765  */
1766 static u8 *
1767 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1768 {
1769 	u8 *addr = *update_ptr;
1770 	u8 *newptr;
1771 	*vmdq = 0;
1772 
1773 	newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1774 	*update_ptr = newptr;
1775 	return addr;
1776 }
1777 
1778 
1779 /*********************************************************************
1780  *  Timer routine
1781  *
1782  *  This routine checks for link status,updates statistics,
1783  *  and runs the watchdog check.
1784  *
1785  **********************************************************************/
1786 
1787 static void
1788 ixgbe_local_timer(void *arg)
1789 {
1790 	struct adapter	*adapter = arg;
1791 	device_t	dev = adapter->dev;
1792 	struct ix_queue *que = adapter->queues;
1793 	u64		queues = 0;
1794 	int		hung = 0;
1795 
1796 	mtx_assert(&adapter->core_mtx, MA_OWNED);
1797 
1798 	/* Check for pluggable optics */
1799 	if (adapter->sfp_probe)
1800 		if (!ixgbe_sfp_probe(adapter))
1801 			goto out; /* Nothing to do */
1802 
1803 	ixgbe_update_link_status(adapter);
1804 	ixgbe_update_stats_counters(adapter);
1805 
1806 	/*
1807 	** Check the TX queues status
1808 	**	- mark hung queues so we don't schedule on them
1809 	**      - watchdog only if all queues show hung
1810 	*/
1811 	for (int i = 0; i < adapter->num_queues; i++, que++) {
1812 		/* Keep track of queues with work for soft irq */
1813 		if (que->txr->busy)
1814 			queues |= ((u64)1 << que->me);
1815 		/*
1816 		** Each time txeof runs without cleaning, but there
1817 		** are uncleaned descriptors it increments busy. If
1818 		** we get to the MAX we declare it hung.
1819 		*/
1820 		if (que->busy == IXGBE_QUEUE_HUNG) {
1821 			++hung;
1822 			/* Mark the queue as inactive */
1823 			adapter->active_queues &= ~((u64)1 << que->me);
1824 			continue;
1825 		} else {
1826 			/* Check if we've come back from hung */
1827 			if ((adapter->active_queues & ((u64)1 << que->me)) == 0)
1828                                 adapter->active_queues |= ((u64)1 << que->me);
1829 		}
1830 		if (que->busy >= IXGBE_MAX_TX_BUSY) {
1831 			device_printf(dev,"Warning queue %d "
1832 			    "appears to be hung!\n", i);
1833 			que->txr->busy = IXGBE_QUEUE_HUNG;
1834 			++hung;
1835 		}
1836 
1837 	}
1838 
1839 	/* Only truly watchdog if all queues show hung */
1840 	if (hung == adapter->num_queues)
1841 		goto watchdog;
1842 	else if (queues != 0) { /* Force an IRQ on queues with work */
1843 		ixgbe_rearm_queues(adapter, queues);
1844 	}
1845 
1846 out:
1847 	callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1848 	return;
1849 
1850 watchdog:
1851 	device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
1852 	adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1853 	adapter->watchdog_events++;
1854 	ixgbe_init_locked(adapter);
1855 }
1856 
1857 /*
1858 ** Note: this routine updates the OS on the link state
1859 **	the real check of the hardware only happens with
1860 **	a link interrupt.
1861 */
1862 static void
1863 ixgbe_update_link_status(struct adapter *adapter)
1864 {
1865 	struct ifnet	*ifp = adapter->ifp;
1866 	device_t dev = adapter->dev;
1867 
1868 
1869 	if (adapter->link_up){
1870 		if (adapter->link_active == FALSE) {
1871 			if (bootverbose)
1872 				device_printf(dev,"Link is up %d Gbps %s \n",
1873 				    ((adapter->link_speed == 128)? 10:1),
1874 				    "Full Duplex");
1875 			adapter->link_active = TRUE;
1876 			/* Update any Flow Control changes */
1877 			ixgbe_fc_enable(&adapter->hw);
1878 			if_link_state_change(ifp, LINK_STATE_UP);
1879 		}
1880 	} else { /* Link down */
1881 		if (adapter->link_active == TRUE) {
1882 			if (bootverbose)
1883 				device_printf(dev,"Link is Down\n");
1884 			if_link_state_change(ifp, LINK_STATE_DOWN);
1885 			adapter->link_active = FALSE;
1886 		}
1887 	}
1888 
1889 	return;
1890 }
1891 
1892 
1893 /*********************************************************************
1894  *
1895  *  This routine disables all traffic on the adapter by issuing a
1896  *  global reset on the MAC and deallocates TX/RX buffers.
1897  *
1898  **********************************************************************/
1899 
1900 static void
1901 ixgbe_stop(void *arg)
1902 {
1903 	struct ifnet   *ifp;
1904 	struct adapter *adapter = arg;
1905 	struct ixgbe_hw *hw = &adapter->hw;
1906 	ifp = adapter->ifp;
1907 
1908 	mtx_assert(&adapter->core_mtx, MA_OWNED);
1909 
1910 	INIT_DEBUGOUT("ixgbe_stop: begin\n");
1911 	ixgbe_disable_intr(adapter);
1912 	callout_stop(&adapter->timer);
1913 
1914 	/* Let the stack know...*/
1915 	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1916 
1917 	ixgbe_reset_hw(hw);
1918 	hw->adapter_stopped = FALSE;
1919 	ixgbe_stop_adapter(hw);
1920 	if (hw->mac.type == ixgbe_mac_82599EB)
1921 		ixgbe_stop_mac_link_on_d3_82599(hw);
1922 	/* Turn off the laser - noop with no optics */
1923 	ixgbe_disable_tx_laser(hw);
1924 
1925 	/* Update the stack */
1926 	adapter->link_up = FALSE;
1927        	ixgbe_update_link_status(adapter);
1928 
1929 	/* reprogram the RAR[0] in case user changed it. */
1930 	ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1931 
1932 	return;
1933 }
1934 
1935 
1936 /*********************************************************************
1937  *
1938  *  Determine hardware revision.
1939  *
1940  **********************************************************************/
1941 static void
1942 ixgbe_identify_hardware(struct adapter *adapter)
1943 {
1944 	device_t        dev = adapter->dev;
1945 	struct ixgbe_hw *hw = &adapter->hw;
1946 
1947 	/* Save off the information about this board */
1948 	hw->vendor_id = pci_get_vendor(dev);
1949 	hw->device_id = pci_get_device(dev);
1950 	hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
1951 	hw->subsystem_vendor_id =
1952 	    pci_read_config(dev, PCIR_SUBVEND_0, 2);
1953 	hw->subsystem_device_id =
1954 	    pci_read_config(dev, PCIR_SUBDEV_0, 2);
1955 
1956 	/*
1957 	** Make sure BUSMASTER is set
1958 	*/
1959 	pci_enable_busmaster(dev);
1960 
1961 	/* We need this here to set the num_segs below */
1962 	ixgbe_set_mac_type(hw);
1963 
1964 	/* Pick up the 82599 and VF settings */
1965 	if (hw->mac.type != ixgbe_mac_82598EB) {
1966 		hw->phy.smart_speed = ixgbe_smart_speed;
1967 		adapter->num_segs = IXGBE_82599_SCATTER;
1968 	} else
1969 		adapter->num_segs = IXGBE_82598_SCATTER;
1970 
1971 	return;
1972 }
1973 
1974 /*********************************************************************
1975  *
1976  *  Determine optic type
1977  *
1978  **********************************************************************/
1979 static void
1980 ixgbe_setup_optics(struct adapter *adapter)
1981 {
1982 	struct ixgbe_hw *hw = &adapter->hw;
1983 	int		layer;
1984 
1985 	layer = ixgbe_get_supported_physical_layer(hw);
1986 
1987 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
1988 		adapter->optics = IFM_10G_T;
1989 		return;
1990 	}
1991 
1992 	if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
1993 		adapter->optics = IFM_1000_T;
1994 		return;
1995 	}
1996 
1997 	if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) {
1998 		adapter->optics = IFM_1000_SX;
1999 		return;
2000 	}
2001 
2002 	if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2003 	    IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2004 		adapter->optics = IFM_10G_LR;
2005 		return;
2006 	}
2007 
2008 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2009 		adapter->optics = IFM_10G_SR;
2010 		return;
2011 	}
2012 
2013 	if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2014 		adapter->optics = IFM_10G_TWINAX;
2015 		return;
2016 	}
2017 
2018 	if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2019 	    IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2020 		adapter->optics = IFM_10G_CX4;
2021 		return;
2022 	}
2023 
2024 	/* If we get here just set the default */
2025 	adapter->optics = IFM_ETHER | IFM_AUTO;
2026 	return;
2027 }
2028 
2029 /*********************************************************************
2030  *
2031  *  Setup the Legacy or MSI Interrupt handler
2032  *
2033  **********************************************************************/
2034 static int
2035 ixgbe_allocate_legacy(struct adapter *adapter)
2036 {
2037 	device_t	dev = adapter->dev;
2038 	struct		ix_queue *que = adapter->queues;
2039 #ifndef IXGBE_LEGACY_TX
2040 	struct tx_ring		*txr = adapter->tx_rings;
2041 #endif
2042 	int		error, rid = 0;
2043 
2044 	/* MSI RID at 1 */
2045 	if (adapter->msix == 1)
2046 		rid = 1;
2047 
2048 	/* We allocate a single interrupt resource */
2049 	adapter->res = bus_alloc_resource_any(dev,
2050             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2051 	if (adapter->res == NULL) {
2052 		device_printf(dev, "Unable to allocate bus resource: "
2053 		    "interrupt\n");
2054 		return (ENXIO);
2055 	}
2056 
2057 	/*
2058 	 * Try allocating a fast interrupt and the associated deferred
2059 	 * processing contexts.
2060 	 */
2061 #ifndef IXGBE_LEGACY_TX
2062 	TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
2063 #endif
2064 	TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2065 	que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2066             taskqueue_thread_enqueue, &que->tq);
2067 	taskqueue_start_threads(&que->tq, 1, PI_NET, "%s ixq",
2068             device_get_nameunit(adapter->dev));
2069 
2070 	/* Tasklets for Link, SFP and Multispeed Fiber */
2071 	TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2072 	TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2073 	TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2074 #ifdef IXGBE_FDIR
2075 	TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2076 #endif
2077 	adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2078 	    taskqueue_thread_enqueue, &adapter->tq);
2079 	taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2080 	    device_get_nameunit(adapter->dev));
2081 
2082 	if ((error = bus_setup_intr(dev, adapter->res,
2083             INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_legacy_irq,
2084             que, &adapter->tag)) != 0) {
2085 		device_printf(dev, "Failed to register fast interrupt "
2086 		    "handler: %d\n", error);
2087 		taskqueue_free(que->tq);
2088 		taskqueue_free(adapter->tq);
2089 		que->tq = NULL;
2090 		adapter->tq = NULL;
2091 		return (error);
2092 	}
2093 	/* For simplicity in the handlers */
2094 	adapter->active_queues = IXGBE_EIMS_ENABLE_MASK;
2095 
2096 	return (0);
2097 }
2098 
2099 
2100 /*********************************************************************
2101  *
2102  *  Setup MSIX Interrupt resources and handlers
2103  *
2104  **********************************************************************/
2105 static int
2106 ixgbe_allocate_msix(struct adapter *adapter)
2107 {
2108 	device_t        dev = adapter->dev;
2109 	struct 		ix_queue *que = adapter->queues;
2110 	struct  	tx_ring *txr = adapter->tx_rings;
2111 	int 		error, rid, vector = 0;
2112 	int		cpu_id = 0;
2113 #ifdef	RSS
2114 	cpuset_t	cpu_mask;
2115 #endif
2116 
2117 #ifdef	RSS
2118 	/*
2119 	 * If we're doing RSS, the number of queues needs to
2120 	 * match the number of RSS buckets that are configured.
2121 	 *
2122 	 * + If there's more queues than RSS buckets, we'll end
2123 	 *   up with queues that get no traffic.
2124 	 *
2125 	 * + If there's more RSS buckets than queues, we'll end
2126 	 *   up having multiple RSS buckets map to the same queue,
2127 	 *   so there'll be some contention.
2128 	 */
2129 	if (adapter->num_queues != rss_getnumbuckets()) {
2130 		device_printf(dev,
2131 		    "%s: number of queues (%d) != number of RSS buckets (%d)"
2132 		    "; performance will be impacted.\n",
2133 		    __func__,
2134 		    adapter->num_queues,
2135 		    rss_getnumbuckets());
2136 	}
2137 #endif
2138 
2139 
2140 
2141 	for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) {
2142 		rid = vector + 1;
2143 		que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2144 		    RF_SHAREABLE | RF_ACTIVE);
2145 		if (que->res == NULL) {
2146 			device_printf(dev,"Unable to allocate"
2147 		    	    " bus resource: que interrupt [%d]\n", vector);
2148 			return (ENXIO);
2149 		}
2150 		/* Set the handler function */
2151 		error = bus_setup_intr(dev, que->res,
2152 		    INTR_TYPE_NET | INTR_MPSAFE, NULL,
2153 		    ixgbe_msix_que, que, &que->tag);
2154 		if (error) {
2155 			que->res = NULL;
2156 			device_printf(dev, "Failed to register QUE handler");
2157 			return (error);
2158 		}
2159 #if __FreeBSD_version >= 800504
2160 		bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2161 #endif
2162 		que->msix = vector;
2163 		adapter->active_queues |= (u64)(1 << que->msix);
2164 #ifdef	RSS
2165 		/*
2166 		 * The queue ID is used as the RSS layer bucket ID.
2167 		 * We look up the queue ID -> RSS CPU ID and select
2168 		 * that.
2169 		 */
2170 		cpu_id = rss_getcpu(i % rss_getnumbuckets());
2171 #else
2172 		/*
2173 		 * Bind the msix vector, and thus the
2174 		 * rings to the corresponding cpu.
2175 		 *
2176 		 * This just happens to match the default RSS round-robin
2177 		 * bucket -> queue -> CPU allocation.
2178 		 */
2179 		if (adapter->num_queues > 1)
2180 			cpu_id = i;
2181 #endif
2182 		if (adapter->num_queues > 1)
2183 			bus_bind_intr(dev, que->res, cpu_id);
2184 
2185 #ifdef	RSS
2186 		device_printf(dev,
2187 		    "Bound RSS bucket %d to CPU %d\n",
2188 		    i, cpu_id);
2189 #else
2190 #if 0 // This is too noisy
2191 		device_printf(dev,
2192 		    "Bound queue %d to cpu %d\n",
2193 		    i, cpu_id);
2194 #endif
2195 #endif
2196 
2197 
2198 #ifndef IXGBE_LEGACY_TX
2199 		TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
2200 #endif
2201 		TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2202 		que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2203 		    taskqueue_thread_enqueue, &que->tq);
2204 #ifdef	RSS
2205 		CPU_SETOF(cpu_id, &cpu_mask);
2206 		taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET,
2207 		    &cpu_mask,
2208 		    "%s (bucket %d)",
2209 		    device_get_nameunit(adapter->dev),
2210 		    cpu_id);
2211 #else
2212 		taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2213 		    device_get_nameunit(adapter->dev));
2214 #endif
2215 	}
2216 
2217 	/* and Link */
2218 	rid = vector + 1;
2219 	adapter->res = bus_alloc_resource_any(dev,
2220     	    SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2221 	if (!adapter->res) {
2222 		device_printf(dev,"Unable to allocate"
2223     	    " bus resource: Link interrupt [%d]\n", rid);
2224 		return (ENXIO);
2225 	}
2226 	/* Set the link handler function */
2227 	error = bus_setup_intr(dev, adapter->res,
2228 	    INTR_TYPE_NET | INTR_MPSAFE, NULL,
2229 	    ixgbe_msix_link, adapter, &adapter->tag);
2230 	if (error) {
2231 		adapter->res = NULL;
2232 		device_printf(dev, "Failed to register LINK handler");
2233 		return (error);
2234 	}
2235 #if __FreeBSD_version >= 800504
2236 	bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2237 #endif
2238 	adapter->vector = vector;
2239 	/* Tasklets for Link, SFP and Multispeed Fiber */
2240 	TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2241 	TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2242 	TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2243 #ifdef IXGBE_FDIR
2244 	TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2245 #endif
2246 	adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2247 	    taskqueue_thread_enqueue, &adapter->tq);
2248 	taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2249 	    device_get_nameunit(adapter->dev));
2250 
2251 	return (0);
2252 }
2253 
2254 /*
2255  * Setup Either MSI/X or MSI
2256  */
2257 static int
2258 ixgbe_setup_msix(struct adapter *adapter)
2259 {
2260 	device_t dev = adapter->dev;
2261 	int rid, want, queues, msgs;
2262 
2263 	/* Override by tuneable */
2264 	if (ixgbe_enable_msix == 0)
2265 		goto msi;
2266 
2267 	/* First try MSI/X */
2268 	msgs = pci_msix_count(dev);
2269 	if (msgs == 0)
2270 		goto msi;
2271 	rid = PCIR_BAR(MSIX_82598_BAR);
2272 	adapter->msix_mem = bus_alloc_resource_any(dev,
2273 	    SYS_RES_MEMORY, &rid, RF_ACTIVE);
2274        	if (adapter->msix_mem == NULL) {
2275 		rid += 4;	/* 82599 maps in higher BAR */
2276 		adapter->msix_mem = bus_alloc_resource_any(dev,
2277 		    SYS_RES_MEMORY, &rid, RF_ACTIVE);
2278 	}
2279        	if (adapter->msix_mem == NULL) {
2280 		/* May not be enabled */
2281 		device_printf(adapter->dev,
2282 		    "Unable to map MSIX table \n");
2283 		goto msi;
2284 	}
2285 
2286 	/* Figure out a reasonable auto config value */
2287 	queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2288 
2289 #ifdef	RSS
2290 	/* If we're doing RSS, clamp at the number of RSS buckets */
2291 	if (queues > rss_getnumbuckets())
2292 		queues = rss_getnumbuckets();
2293 #endif
2294 
2295 	if (ixgbe_num_queues != 0)
2296 		queues = ixgbe_num_queues;
2297 
2298 	/* reflect correct sysctl value */
2299 	ixgbe_num_queues = queues;
2300 
2301 	/*
2302 	** Want one vector (RX/TX pair) per queue
2303 	** plus an additional for Link.
2304 	*/
2305 	want = queues + 1;
2306 	if (msgs >= want)
2307 		msgs = want;
2308 	else {
2309                	device_printf(adapter->dev,
2310 		    "MSIX Configuration Problem, "
2311 		    "%d vectors but %d queues wanted!\n",
2312 		    msgs, want);
2313 		goto msi;
2314 	}
2315 	if ((pci_alloc_msix(dev, &msgs) == 0) && (msgs == want)) {
2316                	device_printf(adapter->dev,
2317 		    "Using MSIX interrupts with %d vectors\n", msgs);
2318 		adapter->num_queues = queues;
2319 		return (msgs);
2320 	}
2321 	/*
2322 	** If MSIX alloc failed or provided us with
2323 	** less than needed, free and fall through to MSI
2324 	*/
2325 	pci_release_msi(dev);
2326 
2327 msi:
2328        	if (adapter->msix_mem != NULL) {
2329 		bus_release_resource(dev, SYS_RES_MEMORY,
2330 		    rid, adapter->msix_mem);
2331 		adapter->msix_mem = NULL;
2332 	}
2333        	msgs = 1;
2334        	if (pci_alloc_msi(dev, &msgs) == 0) {
2335                	device_printf(adapter->dev,"Using an MSI interrupt\n");
2336 		return (msgs);
2337 	}
2338 	device_printf(adapter->dev,"Using a Legacy interrupt\n");
2339 	return (0);
2340 }
2341 
2342 
2343 static int
2344 ixgbe_allocate_pci_resources(struct adapter *adapter)
2345 {
2346 	int             rid;
2347 	device_t        dev = adapter->dev;
2348 
2349 	rid = PCIR_BAR(0);
2350 	adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2351 	    &rid, RF_ACTIVE);
2352 
2353 	if (!(adapter->pci_mem)) {
2354 		device_printf(dev,"Unable to allocate bus resource: memory\n");
2355 		return (ENXIO);
2356 	}
2357 
2358 	adapter->osdep.mem_bus_space_tag =
2359 		rman_get_bustag(adapter->pci_mem);
2360 	adapter->osdep.mem_bus_space_handle =
2361 		rman_get_bushandle(adapter->pci_mem);
2362 	adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2363 
2364 	/* Legacy defaults */
2365 	adapter->num_queues = 1;
2366 	adapter->hw.back = &adapter->osdep;
2367 
2368 	/*
2369 	** Now setup MSI or MSI/X, should
2370 	** return us the number of supported
2371 	** vectors. (Will be 1 for MSI)
2372 	*/
2373 	adapter->msix = ixgbe_setup_msix(adapter);
2374 	return (0);
2375 }
2376 
2377 static void
2378 ixgbe_free_pci_resources(struct adapter * adapter)
2379 {
2380 	struct 		ix_queue *que = adapter->queues;
2381 	device_t	dev = adapter->dev;
2382 	int		rid, memrid;
2383 
2384 	if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2385 		memrid = PCIR_BAR(MSIX_82598_BAR);
2386 	else
2387 		memrid = PCIR_BAR(MSIX_82599_BAR);
2388 
2389 	/*
2390 	** There is a slight possibility of a failure mode
2391 	** in attach that will result in entering this function
2392 	** before interrupt resources have been initialized, and
2393 	** in that case we do not want to execute the loops below
2394 	** We can detect this reliably by the state of the adapter
2395 	** res pointer.
2396 	*/
2397 	if (adapter->res == NULL)
2398 		goto mem;
2399 
2400 	/*
2401 	**  Release all msix queue resources:
2402 	*/
2403 	for (int i = 0; i < adapter->num_queues; i++, que++) {
2404 		rid = que->msix + 1;
2405 		if (que->tag != NULL) {
2406 			bus_teardown_intr(dev, que->res, que->tag);
2407 			que->tag = NULL;
2408 		}
2409 		if (que->res != NULL)
2410 			bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2411 	}
2412 
2413 
2414 	/* Clean the Legacy or Link interrupt last */
2415 	if (adapter->vector) /* we are doing MSIX */
2416 		rid = adapter->vector + 1;
2417 	else
2418 		(adapter->msix != 0) ? (rid = 1):(rid = 0);
2419 
2420 	if (adapter->tag != NULL) {
2421 		bus_teardown_intr(dev, adapter->res, adapter->tag);
2422 		adapter->tag = NULL;
2423 	}
2424 	if (adapter->res != NULL)
2425 		bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2426 
2427 mem:
2428 	if (adapter->msix)
2429 		pci_release_msi(dev);
2430 
2431 	if (adapter->msix_mem != NULL)
2432 		bus_release_resource(dev, SYS_RES_MEMORY,
2433 		    memrid, adapter->msix_mem);
2434 
2435 	if (adapter->pci_mem != NULL)
2436 		bus_release_resource(dev, SYS_RES_MEMORY,
2437 		    PCIR_BAR(0), adapter->pci_mem);
2438 
2439 	return;
2440 }
2441 
2442 /*********************************************************************
2443  *
2444  *  Setup networking device structure and register an interface.
2445  *
2446  **********************************************************************/
2447 static int
2448 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2449 {
2450 	struct ifnet   *ifp;
2451 
2452 	INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2453 
2454 	ifp = adapter->ifp = if_alloc(IFT_ETHER);
2455 	if (ifp == NULL) {
2456 		device_printf(dev, "can not allocate ifnet structure\n");
2457 		return (-1);
2458 	}
2459 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2460 	ifp->if_baudrate = IF_Gbps(10);
2461 	ifp->if_init = ixgbe_init;
2462 	ifp->if_softc = adapter;
2463 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2464 	ifp->if_ioctl = ixgbe_ioctl;
2465 #if __FreeBSD_version >= 1100036
2466 	if_setgetcounterfn(ifp, ixgbe_get_counter);
2467 #endif
2468 #ifndef IXGBE_LEGACY_TX
2469 	ifp->if_transmit = ixgbe_mq_start;
2470 	ifp->if_qflush = ixgbe_qflush;
2471 #else
2472 	ifp->if_start = ixgbe_start;
2473 	IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2);
2474 	ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 2;
2475 	IFQ_SET_READY(&ifp->if_snd);
2476 #endif
2477 
2478 	ether_ifattach(ifp, adapter->hw.mac.addr);
2479 
2480 	adapter->max_frame_size =
2481 	    ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2482 
2483 	/*
2484 	 * Tell the upper layer(s) we support long frames.
2485 	 */
2486 	ifp->if_hdrlen = sizeof(struct ether_vlan_header);
2487 
2488 	ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2489 	ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2490 	ifp->if_capabilities |= IFCAP_LRO;
2491 	ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2492 			     |  IFCAP_VLAN_HWTSO
2493 			     |  IFCAP_VLAN_MTU
2494 			     |  IFCAP_HWSTATS;
2495 	ifp->if_capenable = ifp->if_capabilities;
2496 
2497 	/*
2498 	** Don't turn this on by default, if vlans are
2499 	** created on another pseudo device (eg. lagg)
2500 	** then vlan events are not passed thru, breaking
2501 	** operation, but with HW FILTER off it works. If
2502 	** using vlans directly on the ixgbe driver you can
2503 	** enable this and get full hardware tag filtering.
2504 	*/
2505 	ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2506 
2507 	/*
2508 	 * Specify the media types supported by this adapter and register
2509 	 * callbacks to update media and link information
2510 	 */
2511 	ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2512 		    ixgbe_media_status);
2513 
2514 	ixgbe_add_media_types(adapter);
2515 
2516 	/* Autoselect media by default */
2517 	ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2518 
2519 	return (0);
2520 }
2521 
2522 static void
2523 ixgbe_add_media_types(struct adapter *adapter)
2524 {
2525 	struct ixgbe_hw *hw = &adapter->hw;
2526 	device_t dev = adapter->dev;
2527 	int layer;
2528 
2529 	layer = ixgbe_get_supported_physical_layer(hw);
2530 
2531 	/* Media types with matching FreeBSD media defines */
2532 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T)
2533 		ifmedia_add(&adapter->media, IFM_ETHER | IFM_10G_T, 0, NULL);
2534 	if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T)
2535 		ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_T, 0, NULL);
2536 	if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX)
2537 		ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, 0, NULL);
2538 
2539 	if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU ||
2540 	    layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA)
2541 		ifmedia_add(&adapter->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2542 
2543 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR)
2544 		ifmedia_add(&adapter->media, IFM_ETHER | IFM_10G_LR, 0, NULL);
2545 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR)
2546 		ifmedia_add(&adapter->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2547 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4)
2548 		ifmedia_add(&adapter->media, IFM_ETHER | IFM_10G_CX4, 0, NULL);
2549 	if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX)
2550 		ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
2551 #if 0
2552 	if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_LX)
2553 		ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_LX, 0, NULL);
2554 #endif
2555 
2556 	/*
2557 	** Other (no matching FreeBSD media type):
2558 	** To workaround this, we'll assign these completely
2559 	** inappropriate media types.
2560 	*/
2561 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) {
2562 		device_printf(dev, "Media supported: 10GbaseKR\n");
2563 		device_printf(dev, "10GbaseKR mapped to 10baseT\n");
2564 		ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
2565 	}
2566 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) {
2567 		device_printf(dev, "Media supported: 10GbaseKX4\n");
2568 		device_printf(dev, "10GbaseKX4 mapped to 10base2\n");
2569 		ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_2, 0, NULL);
2570 	}
2571 	if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) {
2572 		device_printf(dev, "Media supported: 1000baseKX\n");
2573 		device_printf(dev, "1000baseKX mapped to 10base5\n");
2574 		ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_5, 0, NULL);
2575 	}
2576 	if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_BX) {
2577 		/* Someday, someone will care about you... */
2578 		device_printf(dev, "Media supported: 1000baseBX\n");
2579 	}
2580 
2581 	/* Very old */
2582 	if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2583 		ifmedia_add(&adapter->media,
2584 		    IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2585 		ifmedia_add(&adapter->media,
2586 		    IFM_ETHER | IFM_1000_T, 0, NULL);
2587 	}
2588 
2589 	ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2590 }
2591 
2592 static void
2593 ixgbe_config_link(struct adapter *adapter)
2594 {
2595 	struct ixgbe_hw *hw = &adapter->hw;
2596 	u32	autoneg, err = 0;
2597 	bool	sfp, negotiate;
2598 
2599 	sfp = ixgbe_is_sfp(hw);
2600 
2601 	if (sfp) {
2602 		if (hw->phy.multispeed_fiber) {
2603 			hw->mac.ops.setup_sfp(hw);
2604 			ixgbe_enable_tx_laser(hw);
2605 			taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2606 		} else
2607 			taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2608 	} else {
2609 		if (hw->mac.ops.check_link)
2610 			err = ixgbe_check_link(hw, &adapter->link_speed,
2611 			    &adapter->link_up, FALSE);
2612 		if (err)
2613 			goto out;
2614 		autoneg = hw->phy.autoneg_advertised;
2615 		if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2616                 	err  = hw->mac.ops.get_link_capabilities(hw,
2617 			    &autoneg, &negotiate);
2618 		if (err)
2619 			goto out;
2620 		if (hw->mac.ops.setup_link)
2621                 	err = hw->mac.ops.setup_link(hw,
2622 			    autoneg, adapter->link_up);
2623 	}
2624 out:
2625 	return;
2626 }
2627 
2628 
2629 /*********************************************************************
2630  *
2631  *  Enable transmit units.
2632  *
2633  **********************************************************************/
2634 static void
2635 ixgbe_initialize_transmit_units(struct adapter *adapter)
2636 {
2637 	struct tx_ring	*txr = adapter->tx_rings;
2638 	struct ixgbe_hw	*hw = &adapter->hw;
2639 
2640 	/* Setup the Base and Length of the Tx Descriptor Ring */
2641 
2642 	for (int i = 0; i < adapter->num_queues; i++, txr++) {
2643 		u64	tdba = txr->txdma.dma_paddr;
2644 		u32	txctrl = 0;
2645 
2646 		IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
2647 		       (tdba & 0x00000000ffffffffULL));
2648 		IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
2649 		IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
2650 		    adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc));
2651 
2652 		/* Setup the HW Tx Head and Tail descriptor pointers */
2653 		IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
2654 		IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
2655 
2656 		/* Cache the tail address */
2657 		txr->tail = IXGBE_TDT(txr->me);
2658 
2659 		/* Set the processing limit */
2660 		txr->process_limit = ixgbe_tx_process_limit;
2661 
2662 		/* Disable Head Writeback */
2663 		switch (hw->mac.type) {
2664 		case ixgbe_mac_82598EB:
2665 			txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
2666 			break;
2667 		case ixgbe_mac_82599EB:
2668 		case ixgbe_mac_X540:
2669 		default:
2670 			txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
2671 			break;
2672                 }
2673 		txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2674 		switch (hw->mac.type) {
2675 		case ixgbe_mac_82598EB:
2676 			IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
2677 			break;
2678 		case ixgbe_mac_82599EB:
2679 		case ixgbe_mac_X540:
2680 		default:
2681 			IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
2682 			break;
2683 		}
2684 
2685 	}
2686 
2687 	if (hw->mac.type != ixgbe_mac_82598EB) {
2688 		u32 dmatxctl, rttdcs;
2689 		dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2690 		dmatxctl |= IXGBE_DMATXCTL_TE;
2691 		IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2692 		/* Disable arbiter to set MTQC */
2693 		rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2694 		rttdcs |= IXGBE_RTTDCS_ARBDIS;
2695 		IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2696 		IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2697 		rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2698 		IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2699 	}
2700 
2701 	return;
2702 }
2703 
2704 static void
2705 ixgbe_initialise_rss_mapping(struct adapter *adapter)
2706 {
2707 	struct ixgbe_hw	*hw = &adapter->hw;
2708 	uint32_t reta;
2709 	int i, j, queue_id;
2710 	uint32_t rss_key[10];
2711 	uint32_t mrqc;
2712 #ifdef	RSS
2713 	uint32_t rss_hash_config;
2714 #endif
2715 
2716 	/* Setup RSS */
2717 	reta = 0;
2718 
2719 #ifdef	RSS
2720 	/* Fetch the configured RSS key */
2721 	rss_getkey((uint8_t *) &rss_key);
2722 #else
2723 	/* set up random bits */
2724 	arc4rand(&rss_key, sizeof(rss_key), 0);
2725 #endif
2726 
2727 	/* Set up the redirection table */
2728 	for (i = 0, j = 0; i < 128; i++, j++) {
2729 		if (j == adapter->num_queues) j = 0;
2730 #ifdef	RSS
2731 		/*
2732 		 * Fetch the RSS bucket id for the given indirection entry.
2733 		 * Cap it at the number of configured buckets (which is
2734 		 * num_queues.)
2735 		 */
2736 		queue_id = rss_get_indirection_to_bucket(i);
2737 		queue_id = queue_id % adapter->num_queues;
2738 #else
2739 		queue_id = (j * 0x11);
2740 #endif
2741 		/*
2742 		 * The low 8 bits are for hash value (n+0);
2743 		 * The next 8 bits are for hash value (n+1), etc.
2744 		 */
2745 		reta = reta >> 8;
2746 		reta = reta | ( ((uint32_t) queue_id) << 24);
2747 		if ((i & 3) == 3) {
2748 			IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2749 			reta = 0;
2750 		}
2751 	}
2752 
2753 	/* Now fill our hash function seeds */
2754 	for (int i = 0; i < 10; i++)
2755 		IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]);
2756 
2757 	/* Perform hash on these packet types */
2758 #ifdef	RSS
2759 	mrqc = IXGBE_MRQC_RSSEN;
2760 	rss_hash_config = rss_gethashconfig();
2761 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
2762 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4;
2763 	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
2764 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP;
2765 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
2766 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6;
2767 	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
2768 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
2769 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
2770 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX;
2771 	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX)
2772 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP;
2773 	if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
2774 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
2775 	if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4_EX)
2776 		device_printf(adapter->dev,
2777 		    "%s: RSS_HASHTYPE_RSS_UDP_IPV4_EX defined, "
2778 		    "but not supported\n", __func__);
2779 	if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
2780 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
2781 	if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX)
2782 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2783 #else
2784 	/*
2785 	 * Disable UDP - IP fragments aren't currently being handled
2786 	 * and so we end up with a mix of 2-tuple and 4-tuple
2787 	 * traffic.
2788 	 */
2789 	mrqc = IXGBE_MRQC_RSSEN
2790 	     | IXGBE_MRQC_RSS_FIELD_IPV4
2791 	     | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
2792 #if 0
2793 	     | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
2794 #endif
2795 	     | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
2796 	     | IXGBE_MRQC_RSS_FIELD_IPV6_EX
2797 	     | IXGBE_MRQC_RSS_FIELD_IPV6
2798 	     | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
2799 #if 0
2800 	     | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
2801 	     | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP
2802 #endif
2803 	;
2804 #endif /* RSS */
2805 	IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2806 }
2807 
2808 
2809 /*********************************************************************
2810  *
2811  *  Setup receive registers and features.
2812  *
2813  **********************************************************************/
2814 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
2815 
2816 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
2817 
2818 static void
2819 ixgbe_initialize_receive_units(struct adapter *adapter)
2820 {
2821 	struct	rx_ring	*rxr = adapter->rx_rings;
2822 	struct ixgbe_hw	*hw = &adapter->hw;
2823 	struct ifnet   *ifp = adapter->ifp;
2824 	u32		bufsz, fctrl, srrctl, rxcsum;
2825 	u32		hlreg;
2826 
2827 
2828 	/*
2829 	 * Make sure receives are disabled while
2830 	 * setting up the descriptor ring
2831 	 */
2832 	ixgbe_disable_rx(hw);
2833 
2834 	/* Enable broadcasts */
2835 	fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2836 	fctrl |= IXGBE_FCTRL_BAM;
2837 	fctrl |= IXGBE_FCTRL_DPF;
2838 	fctrl |= IXGBE_FCTRL_PMCF;
2839 	IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2840 
2841 	/* Set for Jumbo Frames? */
2842 	hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2843 	if (ifp->if_mtu > ETHERMTU)
2844 		hlreg |= IXGBE_HLREG0_JUMBOEN;
2845 	else
2846 		hlreg &= ~IXGBE_HLREG0_JUMBOEN;
2847 #ifdef DEV_NETMAP
2848 	/* crcstrip is conditional in netmap (in RDRXCTL too ?) */
2849 	if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
2850 		hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
2851 	else
2852 		hlreg |= IXGBE_HLREG0_RXCRCSTRP;
2853 #endif /* DEV_NETMAP */
2854 	IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
2855 
2856 	bufsz = (adapter->rx_mbuf_sz +
2857 	    BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2858 
2859 	for (int i = 0; i < adapter->num_queues; i++, rxr++) {
2860 		u64 rdba = rxr->rxdma.dma_paddr;
2861 
2862 		/* Setup the Base and Length of the Rx Descriptor Ring */
2863 		IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
2864 			       (rdba & 0x00000000ffffffffULL));
2865 		IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
2866 		IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
2867 		    adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
2868 
2869 		/* Set up the SRRCTL register */
2870 		srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
2871 		srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
2872 		srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
2873 		srrctl |= bufsz;
2874 		srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2875 
2876 		/*
2877 		 * Set DROP_EN iff we have no flow control and >1 queue.
2878 		 * Note that srrctl was cleared shortly before during reset,
2879 		 * so we do not need to clear the bit, but do it just in case
2880 		 * this code is moved elsewhere.
2881 		 */
2882 		if (adapter->num_queues > 1 &&
2883 		    adapter->hw.fc.requested_mode == ixgbe_fc_none) {
2884 			srrctl |= IXGBE_SRRCTL_DROP_EN;
2885 		} else {
2886 			srrctl &= ~IXGBE_SRRCTL_DROP_EN;
2887 		}
2888 
2889 		IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
2890 
2891 		/* Setup the HW Rx Head and Tail Descriptor Pointers */
2892 		IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
2893 		IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
2894 
2895 		/* Set the processing limit */
2896 		rxr->process_limit = ixgbe_rx_process_limit;
2897 
2898 		/* Set the driver rx tail address */
2899 		rxr->tail =  IXGBE_RDT(rxr->me);
2900 	}
2901 
2902 	if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
2903 		u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
2904 			      IXGBE_PSRTYPE_UDPHDR |
2905 			      IXGBE_PSRTYPE_IPV4HDR |
2906 			      IXGBE_PSRTYPE_IPV6HDR;
2907 		IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
2908 	}
2909 
2910 	rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2911 
2912 	ixgbe_initialise_rss_mapping(adapter);
2913 
2914 	if (adapter->num_queues > 1) {
2915 		/* RSS and RX IPP Checksum are mutually exclusive */
2916 		rxcsum |= IXGBE_RXCSUM_PCSD;
2917 	}
2918 
2919 	if (ifp->if_capenable & IFCAP_RXCSUM)
2920 		rxcsum |= IXGBE_RXCSUM_PCSD;
2921 
2922 	if (!(rxcsum & IXGBE_RXCSUM_PCSD))
2923 		rxcsum |= IXGBE_RXCSUM_IPPCSE;
2924 
2925 	IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2926 
2927 	return;
2928 }
2929 
2930 
2931 /*
2932 ** This routine is run via an vlan config EVENT,
2933 ** it enables us to use the HW Filter table since
2934 ** we can get the vlan id. This just creates the
2935 ** entry in the soft version of the VFTA, init will
2936 ** repopulate the real table.
2937 */
2938 static void
2939 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
2940 {
2941 	struct adapter	*adapter = ifp->if_softc;
2942 	u16		index, bit;
2943 
2944 	if (ifp->if_softc !=  arg)   /* Not our event */
2945 		return;
2946 
2947 	if ((vtag == 0) || (vtag > 4095))	/* Invalid */
2948 		return;
2949 
2950 	IXGBE_CORE_LOCK(adapter);
2951 	index = (vtag >> 5) & 0x7F;
2952 	bit = vtag & 0x1F;
2953 	adapter->shadow_vfta[index] |= (1 << bit);
2954 	++adapter->num_vlans;
2955 	ixgbe_setup_vlan_hw_support(adapter);
2956 	IXGBE_CORE_UNLOCK(adapter);
2957 }
2958 
2959 /*
2960 ** This routine is run via an vlan
2961 ** unconfig EVENT, remove our entry
2962 ** in the soft vfta.
2963 */
2964 static void
2965 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
2966 {
2967 	struct adapter	*adapter = ifp->if_softc;
2968 	u16		index, bit;
2969 
2970 	if (ifp->if_softc !=  arg)
2971 		return;
2972 
2973 	if ((vtag == 0) || (vtag > 4095))	/* Invalid */
2974 		return;
2975 
2976 	IXGBE_CORE_LOCK(adapter);
2977 	index = (vtag >> 5) & 0x7F;
2978 	bit = vtag & 0x1F;
2979 	adapter->shadow_vfta[index] &= ~(1 << bit);
2980 	--adapter->num_vlans;
2981 	/* Re-init to load the changes */
2982 	ixgbe_setup_vlan_hw_support(adapter);
2983 	IXGBE_CORE_UNLOCK(adapter);
2984 }
2985 
2986 static void
2987 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
2988 {
2989 	struct ifnet 	*ifp = adapter->ifp;
2990 	struct ixgbe_hw *hw = &adapter->hw;
2991 	struct rx_ring	*rxr;
2992 	u32		ctrl;
2993 
2994 
2995 	/*
2996 	** We get here thru init_locked, meaning
2997 	** a soft reset, this has already cleared
2998 	** the VFTA and other state, so if there
2999 	** have been no vlan's registered do nothing.
3000 	*/
3001 	if (adapter->num_vlans == 0)
3002 		return;
3003 
3004 	/* Setup the queues for vlans */
3005 	for (int i = 0; i < adapter->num_queues; i++) {
3006 		rxr = &adapter->rx_rings[i];
3007 		/* On 82599 the VLAN enable is per/queue in RXDCTL */
3008 		if (hw->mac.type != ixgbe_mac_82598EB) {
3009 			ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
3010 			ctrl |= IXGBE_RXDCTL_VME;
3011 			IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
3012 		}
3013 		rxr->vtag_strip = TRUE;
3014 	}
3015 
3016 	if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
3017 		return;
3018 	/*
3019 	** A soft reset zero's out the VFTA, so
3020 	** we need to repopulate it now.
3021 	*/
3022 	for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
3023 		if (adapter->shadow_vfta[i] != 0)
3024 			IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
3025 			    adapter->shadow_vfta[i]);
3026 
3027 	ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3028 	/* Enable the Filter Table if enabled */
3029 	if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
3030 		ctrl &= ~IXGBE_VLNCTRL_CFIEN;
3031 		ctrl |= IXGBE_VLNCTRL_VFE;
3032 	}
3033 	if (hw->mac.type == ixgbe_mac_82598EB)
3034 		ctrl |= IXGBE_VLNCTRL_VME;
3035 	IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
3036 }
3037 
3038 static void
3039 ixgbe_enable_intr(struct adapter *adapter)
3040 {
3041 	struct ixgbe_hw	*hw = &adapter->hw;
3042 	struct ix_queue	*que = adapter->queues;
3043 	u32		mask, fwsm;
3044 
3045 	mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
3046 	/* Enable Fan Failure detection */
3047 	if (hw->device_id == IXGBE_DEV_ID_82598AT)
3048 		    mask |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw);
3049 
3050 	switch (adapter->hw.mac.type) {
3051 		case ixgbe_mac_82599EB:
3052 			mask |= IXGBE_EIMS_ECC;
3053 			/* Temperature sensor on some adapters */
3054 			mask |= IXGBE_EIMS_GPI_SDP0_BY_MAC(hw);
3055 			/* SFP+ (RX_LOS_N & MOD_ABS_N) */
3056 			mask |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw);
3057 			mask |= IXGBE_EIMS_GPI_SDP2_BY_MAC(hw);
3058 #ifdef IXGBE_FDIR
3059 			mask |= IXGBE_EIMS_FLOW_DIR;
3060 #endif
3061 			break;
3062 		case ixgbe_mac_X540:
3063 		case ixgbe_mac_X550:
3064 		case ixgbe_mac_X550EM_a:
3065 		case ixgbe_mac_X550EM_x:
3066 			/* Detect if Thermal Sensor is enabled */
3067 			fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM);
3068 			if (fwsm & IXGBE_FWSM_TS_ENABLED)
3069 				mask |= IXGBE_EIMS_TS;
3070 			/* XXX: Which SFP mode line does this look at? */
3071 			if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP)
3072 				mask |= IXGBE_EIMS_GPI_SDP0_BY_MAC(hw);
3073 			mask |= IXGBE_EIMS_ECC;
3074 #ifdef IXGBE_FDIR
3075 			mask |= IXGBE_EIMS_FLOW_DIR;
3076 #endif
3077 		/* falls through */
3078 		default:
3079 			break;
3080 	}
3081 
3082 	IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
3083 
3084 	/* With RSS we use auto clear */
3085 	if (adapter->msix_mem) {
3086 		mask = IXGBE_EIMS_ENABLE_MASK;
3087 		/* Don't autoclear Link */
3088 		mask &= ~IXGBE_EIMS_OTHER;
3089 		mask &= ~IXGBE_EIMS_LSC;
3090 		IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
3091 	}
3092 
3093 	/*
3094 	** Now enable all queues, this is done separately to
3095 	** allow for handling the extended (beyond 32) MSIX
3096 	** vectors that can be used by 82599
3097 	*/
3098         for (int i = 0; i < adapter->num_queues; i++, que++)
3099                 ixgbe_enable_queue(adapter, que->msix);
3100 
3101 	IXGBE_WRITE_FLUSH(hw);
3102 
3103 	return;
3104 }
3105 
3106 static void
3107 ixgbe_disable_intr(struct adapter *adapter)
3108 {
3109 	if (adapter->msix_mem)
3110 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
3111 	if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
3112 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
3113 	} else {
3114 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
3115 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
3116 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
3117 	}
3118 	IXGBE_WRITE_FLUSH(&adapter->hw);
3119 	return;
3120 }
3121 
3122 /*
3123 ** Get the width and transaction speed of
3124 ** the slot this adapter is plugged into.
3125 */
3126 static void
3127 ixgbe_get_slot_info(struct ixgbe_hw *hw)
3128 {
3129 	device_t		dev = ((struct ixgbe_osdep *)hw->back)->dev;
3130 	struct ixgbe_mac_info	*mac = &hw->mac;
3131 	u16			link;
3132 	u32			offset;
3133 
3134 	/* For most devices simply call the shared code routine */
3135 	if (hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) {
3136 		ixgbe_get_bus_info(hw);
3137 		/* These devices don't use PCI-E */
3138 		if (hw->mac.type == ixgbe_mac_X550EM_x
3139 		    || hw->mac.type == ixgbe_mac_X550EM_a)
3140 			return;
3141 		goto display;
3142 	}
3143 
3144 	/*
3145 	** For the Quad port adapter we need to parse back
3146 	** up the PCI tree to find the speed of the expansion
3147 	** slot into which this adapter is plugged. A bit more work.
3148 	*/
3149 	dev = device_get_parent(device_get_parent(dev));
3150 #ifdef IXGBE_DEBUG
3151 	device_printf(dev, "parent pcib = %x,%x,%x\n",
3152 	    pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev));
3153 #endif
3154 	dev = device_get_parent(device_get_parent(dev));
3155 #ifdef IXGBE_DEBUG
3156 	device_printf(dev, "slot pcib = %x,%x,%x\n",
3157 	    pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev));
3158 #endif
3159 	/* Now get the PCI Express Capabilities offset */
3160 	pci_find_cap(dev, PCIY_EXPRESS, &offset);
3161 	/* ...and read the Link Status Register */
3162 	link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
3163 	switch (link & IXGBE_PCI_LINK_WIDTH) {
3164 	case IXGBE_PCI_LINK_WIDTH_1:
3165 		hw->bus.width = ixgbe_bus_width_pcie_x1;
3166 		break;
3167 	case IXGBE_PCI_LINK_WIDTH_2:
3168 		hw->bus.width = ixgbe_bus_width_pcie_x2;
3169 		break;
3170 	case IXGBE_PCI_LINK_WIDTH_4:
3171 		hw->bus.width = ixgbe_bus_width_pcie_x4;
3172 		break;
3173 	case IXGBE_PCI_LINK_WIDTH_8:
3174 		hw->bus.width = ixgbe_bus_width_pcie_x8;
3175 		break;
3176 	default:
3177 		hw->bus.width = ixgbe_bus_width_unknown;
3178 		break;
3179 	}
3180 
3181 	switch (link & IXGBE_PCI_LINK_SPEED) {
3182 	case IXGBE_PCI_LINK_SPEED_2500:
3183 		hw->bus.speed = ixgbe_bus_speed_2500;
3184 		break;
3185 	case IXGBE_PCI_LINK_SPEED_5000:
3186 		hw->bus.speed = ixgbe_bus_speed_5000;
3187 		break;
3188 	case IXGBE_PCI_LINK_SPEED_8000:
3189 		hw->bus.speed = ixgbe_bus_speed_8000;
3190 		break;
3191 	default:
3192 		hw->bus.speed = ixgbe_bus_speed_unknown;
3193 		break;
3194 	}
3195 
3196 	mac->ops.set_lan_id(hw);
3197 
3198 display:
3199 	device_printf(dev,"PCI Express Bus: Speed %s %s\n",
3200 	    ((hw->bus.speed == ixgbe_bus_speed_8000) ? "8.0GT/s":
3201 	    (hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0GT/s":
3202 	    (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5GT/s":"Unknown"),
3203 	    (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
3204 	    (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
3205 	    (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
3206 	    ("Unknown"));
3207 
3208 	if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) &&
3209 	    ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
3210 	    (hw->bus.speed == ixgbe_bus_speed_2500))) {
3211 		device_printf(dev, "PCI-Express bandwidth available"
3212 		    " for this card\n     is not sufficient for"
3213 		    " optimal performance.\n");
3214 		device_printf(dev, "For optimal performance a x8 "
3215 		    "PCIE, or x4 PCIE Gen2 slot is required.\n");
3216         }
3217 	if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) &&
3218 	    ((hw->bus.width <= ixgbe_bus_width_pcie_x8) &&
3219 	    (hw->bus.speed < ixgbe_bus_speed_8000))) {
3220 		device_printf(dev, "PCI-Express bandwidth available"
3221 		    " for this card\n     is not sufficient for"
3222 		    " optimal performance.\n");
3223 		device_printf(dev, "For optimal performance a x8 "
3224 		    "PCIE Gen3 slot is required.\n");
3225         }
3226 
3227 	return;
3228 }
3229 
3230 
3231 /*
3232 ** Setup the correct IVAR register for a particular MSIX interrupt
3233 **   (yes this is all very magic and confusing :)
3234 **  - entry is the register array entry
3235 **  - vector is the MSIX vector for this queue
3236 **  - type is RX/TX/MISC
3237 */
3238 static void
3239 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
3240 {
3241 	struct ixgbe_hw *hw = &adapter->hw;
3242 	u32 ivar, index;
3243 
3244 	vector |= IXGBE_IVAR_ALLOC_VAL;
3245 
3246 	switch (hw->mac.type) {
3247 
3248 	case ixgbe_mac_82598EB:
3249 		if (type == -1)
3250 			entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
3251 		else
3252 			entry += (type * 64);
3253 		index = (entry >> 2) & 0x1F;
3254 		ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
3255 		ivar &= ~(0xFF << (8 * (entry & 0x3)));
3256 		ivar |= (vector << (8 * (entry & 0x3)));
3257 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
3258 		break;
3259 
3260 	case ixgbe_mac_82599EB:
3261 	case ixgbe_mac_X540:
3262 	case ixgbe_mac_X550:
3263 	case ixgbe_mac_X550EM_a:
3264 	case ixgbe_mac_X550EM_x:
3265 		if (type == -1) { /* MISC IVAR */
3266 			index = (entry & 1) * 8;
3267 			ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
3268 			ivar &= ~(0xFF << index);
3269 			ivar |= (vector << index);
3270 			IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
3271 		} else {	/* RX/TX IVARS */
3272 			index = (16 * (entry & 1)) + (8 * type);
3273 			ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
3274 			ivar &= ~(0xFF << index);
3275 			ivar |= (vector << index);
3276 			IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
3277 		}
3278 
3279 	default:
3280 		break;
3281 	}
3282 }
3283 
3284 static void
3285 ixgbe_configure_ivars(struct adapter *adapter)
3286 {
3287 	struct  ix_queue *que = adapter->queues;
3288 	u32 newitr;
3289 
3290 	if (ixgbe_max_interrupt_rate > 0)
3291 		newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
3292 	else
3293 		newitr = 0;
3294 
3295         for (int i = 0; i < adapter->num_queues; i++, que++) {
3296 		/* First the RX queue entry */
3297                 ixgbe_set_ivar(adapter, i, que->msix, 0);
3298 		/* ... and the TX */
3299 		ixgbe_set_ivar(adapter, i, que->msix, 1);
3300 		/* Set an Initial EITR value */
3301                 IXGBE_WRITE_REG(&adapter->hw,
3302                     IXGBE_EITR(que->msix), newitr);
3303 	}
3304 
3305 	/* For the Link interrupt */
3306         ixgbe_set_ivar(adapter, 1, adapter->vector, -1);
3307 }
3308 
3309 /*
3310 ** ixgbe_sfp_probe - called in the local timer to
3311 ** determine if a port had optics inserted.
3312 */
3313 static bool ixgbe_sfp_probe(struct adapter *adapter)
3314 {
3315 	struct ixgbe_hw	*hw = &adapter->hw;
3316 	device_t	dev = adapter->dev;
3317 	bool		result = FALSE;
3318 
3319 	if ((hw->phy.type == ixgbe_phy_nl) &&
3320 	    (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
3321 		s32 ret = hw->phy.ops.identify_sfp(hw);
3322 		if (ret)
3323                         goto out;
3324 		ret = hw->phy.ops.reset(hw);
3325 		if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
3326 			device_printf(dev,"Unsupported SFP+ module detected!");
3327 			printf(" Reload driver with supported module.\n");
3328 			adapter->sfp_probe = FALSE;
3329                         goto out;
3330 		} else
3331 			device_printf(dev,"SFP+ module detected!\n");
3332 		/* We now have supported optics */
3333 		adapter->sfp_probe = FALSE;
3334 		/* Set the optics type so system reports correctly */
3335 		ixgbe_setup_optics(adapter);
3336 		result = TRUE;
3337 	}
3338 out:
3339 	return (result);
3340 }
3341 
3342 /*
3343 ** Tasklet handler for MSIX Link interrupts
3344 **  - do outside interrupt since it might sleep
3345 */
3346 static void
3347 ixgbe_handle_link(void *context, int pending)
3348 {
3349 	struct adapter  *adapter = context;
3350 
3351 	ixgbe_check_link(&adapter->hw,
3352 	    &adapter->link_speed, &adapter->link_up, 0);
3353        	ixgbe_update_link_status(adapter);
3354 }
3355 
3356 /*
3357 ** Tasklet for handling SFP module interrupts
3358 */
3359 static void
3360 ixgbe_handle_mod(void *context, int pending)
3361 {
3362 	struct adapter  *adapter = context;
3363 	struct ixgbe_hw *hw = &adapter->hw;
3364 	device_t	dev = adapter->dev;
3365 	u32 err;
3366 
3367 	err = hw->phy.ops.identify_sfp(hw);
3368 	if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
3369 		device_printf(dev,
3370 		    "Unsupported SFP+ module type was detected.\n");
3371 		return;
3372 	}
3373 	err = hw->mac.ops.setup_sfp(hw);
3374 	if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
3375 		device_printf(dev,
3376 		    "Setup failure - unsupported SFP+ module type.\n");
3377 		return;
3378 	}
3379 	taskqueue_enqueue(adapter->tq, &adapter->msf_task);
3380 	return;
3381 }
3382 
3383 
3384 /*
3385 ** Tasklet for handling MSF (multispeed fiber) interrupts
3386 */
3387 static void
3388 ixgbe_handle_msf(void *context, int pending)
3389 {
3390 	struct adapter  *adapter = context;
3391 	struct ixgbe_hw *hw = &adapter->hw;
3392 	u32 autoneg;
3393 	bool negotiate;
3394 	int err;
3395 
3396 	err = hw->phy.ops.identify_sfp(hw);
3397 	if (!err) {
3398 		ixgbe_setup_optics(adapter);
3399 		INIT_DEBUGOUT1("ixgbe_sfp_probe: flags: %X\n", adapter->optics);
3400 	}
3401 
3402 	autoneg = hw->phy.autoneg_advertised;
3403 	if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
3404 		hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
3405 	if (hw->mac.ops.setup_link)
3406 		hw->mac.ops.setup_link(hw, autoneg, TRUE);
3407 
3408 	ifmedia_removeall(&adapter->media);
3409 	ixgbe_add_media_types(adapter);
3410 	return;
3411 }
3412 
3413 #ifdef IXGBE_FDIR
3414 /*
3415 ** Tasklet for reinitializing the Flow Director filter table
3416 */
3417 static void
3418 ixgbe_reinit_fdir(void *context, int pending)
3419 {
3420 	struct adapter  *adapter = context;
3421 	struct ifnet   *ifp = adapter->ifp;
3422 
3423 	if (adapter->fdir_reinit != 1) /* Shouldn't happen */
3424 		return;
3425 	ixgbe_reinit_fdir_tables_82599(&adapter->hw);
3426 	adapter->fdir_reinit = 0;
3427 	/* re-enable flow director interrupts */
3428 	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
3429 	/* Restart the interface */
3430 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
3431 	return;
3432 }
3433 #endif
3434 
3435 /**********************************************************************
3436  *
3437  *  Update the board statistics counters.
3438  *
3439  **********************************************************************/
3440 static void
3441 ixgbe_update_stats_counters(struct adapter *adapter)
3442 {
3443 	struct ixgbe_hw *hw = &adapter->hw;
3444 	u32 missed_rx = 0, bprc, lxon, lxoff, total;
3445 	u64 total_missed_rx = 0;
3446 
3447 	adapter->stats.pf.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
3448 	adapter->stats.pf.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
3449 	adapter->stats.pf.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
3450 	adapter->stats.pf.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
3451 
3452 	/*
3453 	** Note: these are for the 8 possible traffic classes,
3454 	**	 which in current implementation is unused,
3455 	**	 therefore only 0 should read real data.
3456 	*/
3457 	for (int i = 0; i < 8; i++) {
3458 		u32 mp;
3459 		mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
3460 		/* missed_rx tallies misses for the gprc workaround */
3461 		missed_rx += mp;
3462 		/* global total per queue */
3463         	adapter->stats.pf.mpc[i] += mp;
3464 		/* total for stats display */
3465 		total_missed_rx += adapter->stats.pf.mpc[i];
3466 		if (hw->mac.type == ixgbe_mac_82598EB) {
3467 			adapter->stats.pf.rnbc[i] +=
3468 			    IXGBE_READ_REG(hw, IXGBE_RNBC(i));
3469 			adapter->stats.pf.qbtc[i] +=
3470 			    IXGBE_READ_REG(hw, IXGBE_QBTC(i));
3471 			adapter->stats.pf.qbrc[i] +=
3472 			    IXGBE_READ_REG(hw, IXGBE_QBRC(i));
3473 			adapter->stats.pf.pxonrxc[i] +=
3474 		    	    IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
3475 		} else
3476 			adapter->stats.pf.pxonrxc[i] +=
3477 		    	    IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
3478 		adapter->stats.pf.pxontxc[i] +=
3479 		    IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
3480 		adapter->stats.pf.pxofftxc[i] +=
3481 		    IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
3482 		if (hw->mac.type != ixgbe_mac_X550EM_x)
3483 			adapter->stats.pf.pxoffrxc[i] +=
3484 			    IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
3485 		adapter->stats.pf.pxon2offc[i] +=
3486 		    IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
3487 	}
3488 	for (int i = 0; i < 16; i++) {
3489 		adapter->stats.pf.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
3490 		adapter->stats.pf.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
3491 		adapter->stats.pf.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
3492 	}
3493 	adapter->stats.pf.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
3494 	adapter->stats.pf.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
3495 	adapter->stats.pf.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
3496 
3497 	/* Hardware workaround, gprc counts missed packets */
3498 	adapter->stats.pf.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
3499 	adapter->stats.pf.gprc -= missed_rx;
3500 
3501 	if (hw->mac.type != ixgbe_mac_82598EB) {
3502 		adapter->stats.pf.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
3503 		    ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
3504 		adapter->stats.pf.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
3505 		    ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
3506 		adapter->stats.pf.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
3507 		    ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
3508 		adapter->stats.pf.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
3509 		adapter->stats.pf.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
3510 	} else {
3511 		adapter->stats.pf.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
3512 		adapter->stats.pf.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
3513 		/* 82598 only has a counter in the high register */
3514 		adapter->stats.pf.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
3515 		adapter->stats.pf.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
3516 		adapter->stats.pf.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
3517 	}
3518 
3519 	/*
3520 	 * Workaround: mprc hardware is incorrectly counting
3521 	 * broadcasts, so for now we subtract those.
3522 	 */
3523 	bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
3524 	adapter->stats.pf.bprc += bprc;
3525 	adapter->stats.pf.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
3526 	if (hw->mac.type == ixgbe_mac_82598EB)
3527 		adapter->stats.pf.mprc -= bprc;
3528 
3529 	adapter->stats.pf.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
3530 	adapter->stats.pf.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
3531 	adapter->stats.pf.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
3532 	adapter->stats.pf.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
3533 	adapter->stats.pf.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
3534 	adapter->stats.pf.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
3535 
3536 	lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
3537 	adapter->stats.pf.lxontxc += lxon;
3538 	lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
3539 	adapter->stats.pf.lxofftxc += lxoff;
3540 	total = lxon + lxoff;
3541 
3542 	adapter->stats.pf.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
3543 	adapter->stats.pf.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
3544 	adapter->stats.pf.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
3545 	adapter->stats.pf.gptc -= total;
3546 	adapter->stats.pf.mptc -= total;
3547 	adapter->stats.pf.ptc64 -= total;
3548 	adapter->stats.pf.gotc -= total * ETHER_MIN_LEN;
3549 
3550 	adapter->stats.pf.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
3551 	adapter->stats.pf.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
3552 	adapter->stats.pf.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
3553 	adapter->stats.pf.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
3554 	adapter->stats.pf.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
3555 	adapter->stats.pf.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
3556 	adapter->stats.pf.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
3557 	adapter->stats.pf.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
3558 	adapter->stats.pf.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
3559 	adapter->stats.pf.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
3560 	adapter->stats.pf.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
3561 	adapter->stats.pf.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
3562 	adapter->stats.pf.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
3563 	adapter->stats.pf.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
3564 	adapter->stats.pf.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
3565 	adapter->stats.pf.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
3566 	adapter->stats.pf.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
3567 	adapter->stats.pf.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
3568 	/* Only read FCOE on 82599 */
3569 	if (hw->mac.type != ixgbe_mac_82598EB) {
3570 		adapter->stats.pf.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
3571 		adapter->stats.pf.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
3572 		adapter->stats.pf.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
3573 		adapter->stats.pf.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
3574 		adapter->stats.pf.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
3575 	}
3576 
3577 	/* Fill out the OS statistics structure */
3578 	IXGBE_SET_IPACKETS(adapter, adapter->stats.pf.gprc);
3579 	IXGBE_SET_OPACKETS(adapter, adapter->stats.pf.gptc);
3580 	IXGBE_SET_IBYTES(adapter, adapter->stats.pf.gorc);
3581 	IXGBE_SET_OBYTES(adapter, adapter->stats.pf.gotc);
3582 	IXGBE_SET_IMCASTS(adapter, adapter->stats.pf.mprc);
3583 	IXGBE_SET_OMCASTS(adapter, adapter->stats.pf.mptc);
3584 	IXGBE_SET_COLLISIONS(adapter, 0);
3585 	IXGBE_SET_IQDROPS(adapter, total_missed_rx);
3586 	IXGBE_SET_IERRORS(adapter, adapter->stats.pf.crcerrs
3587 	    + adapter->stats.pf.rlec);
3588 }
3589 
3590 #if __FreeBSD_version >= 1100036
3591 static uint64_t
3592 ixgbe_get_counter(struct ifnet *ifp, ift_counter cnt)
3593 {
3594 	struct adapter *adapter;
3595 
3596 	adapter = if_getsoftc(ifp);
3597 
3598 	switch (cnt) {
3599 	case IFCOUNTER_IPACKETS:
3600 		return (adapter->ipackets);
3601 	case IFCOUNTER_OPACKETS:
3602 		return (adapter->opackets);
3603 	case IFCOUNTER_IBYTES:
3604 		return (adapter->ibytes);
3605 	case IFCOUNTER_OBYTES:
3606 		return (adapter->obytes);
3607 	case IFCOUNTER_IMCASTS:
3608 		return (adapter->imcasts);
3609 	case IFCOUNTER_OMCASTS:
3610 		return (adapter->omcasts);
3611 	case IFCOUNTER_COLLISIONS:
3612 		return (0);
3613 	case IFCOUNTER_IQDROPS:
3614 		return (adapter->iqdrops);
3615 	case IFCOUNTER_IERRORS:
3616 		return (adapter->ierrors);
3617 	default:
3618 		return (if_get_counter_default(ifp, cnt));
3619 	}
3620 }
3621 #endif
3622 
3623 /** ixgbe_sysctl_tdh_handler - Handler function
3624  *  Retrieves the TDH value from the hardware
3625  */
3626 static int
3627 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
3628 {
3629 	int error;
3630 
3631 	struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
3632 	if (!txr) return 0;
3633 
3634 	unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
3635 	error = sysctl_handle_int(oidp, &val, 0, req);
3636 	if (error || !req->newptr)
3637 		return error;
3638 	return 0;
3639 }
3640 
3641 /** ixgbe_sysctl_tdt_handler - Handler function
3642  *  Retrieves the TDT value from the hardware
3643  */
3644 static int
3645 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
3646 {
3647 	int error;
3648 
3649 	struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
3650 	if (!txr) return 0;
3651 
3652 	unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
3653 	error = sysctl_handle_int(oidp, &val, 0, req);
3654 	if (error || !req->newptr)
3655 		return error;
3656 	return 0;
3657 }
3658 
3659 /** ixgbe_sysctl_rdh_handler - Handler function
3660  *  Retrieves the RDH value from the hardware
3661  */
3662 static int
3663 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
3664 {
3665 	int error;
3666 
3667 	struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
3668 	if (!rxr) return 0;
3669 
3670 	unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
3671 	error = sysctl_handle_int(oidp, &val, 0, req);
3672 	if (error || !req->newptr)
3673 		return error;
3674 	return 0;
3675 }
3676 
3677 /** ixgbe_sysctl_rdt_handler - Handler function
3678  *  Retrieves the RDT value from the hardware
3679  */
3680 static int
3681 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
3682 {
3683 	int error;
3684 
3685 	struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
3686 	if (!rxr) return 0;
3687 
3688 	unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
3689 	error = sysctl_handle_int(oidp, &val, 0, req);
3690 	if (error || !req->newptr)
3691 		return error;
3692 	return 0;
3693 }
3694 
3695 static int
3696 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
3697 {
3698 	int error;
3699 	struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
3700 	unsigned int reg, usec, rate;
3701 
3702 	reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
3703 	usec = ((reg & 0x0FF8) >> 3);
3704 	if (usec > 0)
3705 		rate = 500000 / usec;
3706 	else
3707 		rate = 0;
3708 	error = sysctl_handle_int(oidp, &rate, 0, req);
3709 	if (error || !req->newptr)
3710 		return error;
3711 	reg &= ~0xfff; /* default, no limitation */
3712 	ixgbe_max_interrupt_rate = 0;
3713 	if (rate > 0 && rate < 500000) {
3714 		if (rate < 1000)
3715 			rate = 1000;
3716 		ixgbe_max_interrupt_rate = rate;
3717 		reg |= ((4000000/rate) & 0xff8 );
3718 	}
3719 	IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
3720 	return 0;
3721 }
3722 
3723 /*
3724  * Add sysctl variables, one per statistic, to the system.
3725  */
3726 static void
3727 ixgbe_add_hw_stats(struct adapter *adapter)
3728 {
3729 	device_t dev = adapter->dev;
3730 
3731 	struct tx_ring *txr = adapter->tx_rings;
3732 	struct rx_ring *rxr = adapter->rx_rings;
3733 
3734 	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
3735 	struct sysctl_oid *tree = device_get_sysctl_tree(dev);
3736 	struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
3737 	struct ixgbe_hw_stats *stats = &adapter->stats.pf;
3738 
3739 	struct sysctl_oid *stat_node, *queue_node;
3740 	struct sysctl_oid_list *stat_list, *queue_list;
3741 
3742 #define QUEUE_NAME_LEN 32
3743 	char namebuf[QUEUE_NAME_LEN];
3744 
3745 	/* Driver Statistics */
3746 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
3747 			CTLFLAG_RD, &adapter->dropped_pkts,
3748 			"Driver dropped packets");
3749 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
3750 			CTLFLAG_RD, &adapter->mbuf_defrag_failed,
3751 			"m_defrag() failed");
3752 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
3753 			CTLFLAG_RD, &adapter->watchdog_events,
3754 			"Watchdog timeouts");
3755 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
3756 			CTLFLAG_RD, &adapter->vector_irq,
3757 			"Link MSIX IRQ Handled");
3758 
3759 	for (int i = 0; i < adapter->num_queues; i++, txr++) {
3760 		snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
3761 		queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
3762 					    CTLFLAG_RD, NULL, "Queue Name");
3763 		queue_list = SYSCTL_CHILDREN(queue_node);
3764 
3765 		SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
3766 				CTLTYPE_UINT | CTLFLAG_RW, &adapter->queues[i],
3767 				sizeof(&adapter->queues[i]),
3768 				ixgbe_sysctl_interrupt_rate_handler, "IU",
3769 				"Interrupt Rate");
3770 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
3771 				CTLFLAG_RD, &(adapter->queues[i].irqs),
3772 				"irqs on this queue");
3773 		SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
3774 				CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
3775 				ixgbe_sysctl_tdh_handler, "IU",
3776 				"Transmit Descriptor Head");
3777 		SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
3778 				CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
3779 				ixgbe_sysctl_tdt_handler, "IU",
3780 				"Transmit Descriptor Tail");
3781 		SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tso_tx",
3782 				CTLFLAG_RD, &txr->tso_tx,
3783 				"TSO");
3784 		SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_tx_dma_setup",
3785 				CTLFLAG_RD, &txr->no_tx_dma_setup,
3786 				"Driver tx dma failure in xmit");
3787 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
3788 				CTLFLAG_RD, &txr->no_desc_avail,
3789 				"Queue No Descriptor Available");
3790 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
3791 				CTLFLAG_RD, &txr->total_packets,
3792 				"Queue Packets Transmitted");
3793 	}
3794 
3795 	for (int i = 0; i < adapter->num_queues; i++, rxr++) {
3796 		snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
3797 		queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
3798 					    CTLFLAG_RD, NULL, "Queue Name");
3799 		queue_list = SYSCTL_CHILDREN(queue_node);
3800 
3801 		struct lro_ctrl *lro = &rxr->lro;
3802 
3803 		snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
3804 		queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
3805 					    CTLFLAG_RD, NULL, "Queue Name");
3806 		queue_list = SYSCTL_CHILDREN(queue_node);
3807 
3808 		SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
3809 				CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
3810 				ixgbe_sysctl_rdh_handler, "IU",
3811 				"Receive Descriptor Head");
3812 		SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
3813 				CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
3814 				ixgbe_sysctl_rdt_handler, "IU",
3815 				"Receive Descriptor Tail");
3816 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
3817 				CTLFLAG_RD, &rxr->rx_packets,
3818 				"Queue Packets Received");
3819 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
3820 				CTLFLAG_RD, &rxr->rx_bytes,
3821 				"Queue Bytes Received");
3822 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_copies",
3823 				CTLFLAG_RD, &rxr->rx_copies,
3824 				"Copied RX Frames");
3825 		SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
3826 				CTLFLAG_RD, &lro->lro_queued, 0,
3827 				"LRO Queued");
3828 		SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
3829 				CTLFLAG_RD, &lro->lro_flushed, 0,
3830 				"LRO Flushed");
3831 	}
3832 
3833 	/* MAC stats get the own sub node */
3834 
3835 	stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
3836 				    CTLFLAG_RD, NULL, "MAC Statistics");
3837 	stat_list = SYSCTL_CHILDREN(stat_node);
3838 
3839 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
3840 			CTLFLAG_RD, &stats->crcerrs,
3841 			"CRC Errors");
3842 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
3843 			CTLFLAG_RD, &stats->illerrc,
3844 			"Illegal Byte Errors");
3845 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
3846 			CTLFLAG_RD, &stats->errbc,
3847 			"Byte Errors");
3848 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
3849 			CTLFLAG_RD, &stats->mspdc,
3850 			"MAC Short Packets Discarded");
3851 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
3852 			CTLFLAG_RD, &stats->mlfc,
3853 			"MAC Local Faults");
3854 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
3855 			CTLFLAG_RD, &stats->mrfc,
3856 			"MAC Remote Faults");
3857 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
3858 			CTLFLAG_RD, &stats->rlec,
3859 			"Receive Length Errors");
3860 
3861 	/* Flow Control stats */
3862 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
3863 			CTLFLAG_RD, &stats->lxontxc,
3864 			"Link XON Transmitted");
3865 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
3866 			CTLFLAG_RD, &stats->lxonrxc,
3867 			"Link XON Received");
3868 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
3869 			CTLFLAG_RD, &stats->lxofftxc,
3870 			"Link XOFF Transmitted");
3871 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
3872 			CTLFLAG_RD, &stats->lxoffrxc,
3873 			"Link XOFF Received");
3874 
3875 	/* Packet Reception Stats */
3876 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
3877 			CTLFLAG_RD, &stats->tor,
3878 			"Total Octets Received");
3879 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
3880 			CTLFLAG_RD, &stats->gorc,
3881 			"Good Octets Received");
3882 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
3883 			CTLFLAG_RD, &stats->tpr,
3884 			"Total Packets Received");
3885 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
3886 			CTLFLAG_RD, &stats->gprc,
3887 			"Good Packets Received");
3888 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
3889 			CTLFLAG_RD, &stats->mprc,
3890 			"Multicast Packets Received");
3891 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
3892 			CTLFLAG_RD, &stats->bprc,
3893 			"Broadcast Packets Received");
3894 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
3895 			CTLFLAG_RD, &stats->prc64,
3896 			"64 byte frames received ");
3897 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
3898 			CTLFLAG_RD, &stats->prc127,
3899 			"65-127 byte frames received");
3900 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
3901 			CTLFLAG_RD, &stats->prc255,
3902 			"128-255 byte frames received");
3903 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
3904 			CTLFLAG_RD, &stats->prc511,
3905 			"256-511 byte frames received");
3906 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
3907 			CTLFLAG_RD, &stats->prc1023,
3908 			"512-1023 byte frames received");
3909 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
3910 			CTLFLAG_RD, &stats->prc1522,
3911 			"1023-1522 byte frames received");
3912 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
3913 			CTLFLAG_RD, &stats->ruc,
3914 			"Receive Undersized");
3915 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
3916 			CTLFLAG_RD, &stats->rfc,
3917 			"Fragmented Packets Received ");
3918 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
3919 			CTLFLAG_RD, &stats->roc,
3920 			"Oversized Packets Received");
3921 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
3922 			CTLFLAG_RD, &stats->rjc,
3923 			"Received Jabber");
3924 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
3925 			CTLFLAG_RD, &stats->mngprc,
3926 			"Management Packets Received");
3927 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
3928 			CTLFLAG_RD, &stats->mngptc,
3929 			"Management Packets Dropped");
3930 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
3931 			CTLFLAG_RD, &stats->xec,
3932 			"Checksum Errors");
3933 
3934 	/* Packet Transmission Stats */
3935 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
3936 			CTLFLAG_RD, &stats->gotc,
3937 			"Good Octets Transmitted");
3938 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
3939 			CTLFLAG_RD, &stats->tpt,
3940 			"Total Packets Transmitted");
3941 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
3942 			CTLFLAG_RD, &stats->gptc,
3943 			"Good Packets Transmitted");
3944 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
3945 			CTLFLAG_RD, &stats->bptc,
3946 			"Broadcast Packets Transmitted");
3947 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
3948 			CTLFLAG_RD, &stats->mptc,
3949 			"Multicast Packets Transmitted");
3950 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
3951 			CTLFLAG_RD, &stats->mngptc,
3952 			"Management Packets Transmitted");
3953 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
3954 			CTLFLAG_RD, &stats->ptc64,
3955 			"64 byte frames transmitted ");
3956 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
3957 			CTLFLAG_RD, &stats->ptc127,
3958 			"65-127 byte frames transmitted");
3959 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
3960 			CTLFLAG_RD, &stats->ptc255,
3961 			"128-255 byte frames transmitted");
3962 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
3963 			CTLFLAG_RD, &stats->ptc511,
3964 			"256-511 byte frames transmitted");
3965 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
3966 			CTLFLAG_RD, &stats->ptc1023,
3967 			"512-1023 byte frames transmitted");
3968 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
3969 			CTLFLAG_RD, &stats->ptc1522,
3970 			"1024-1522 byte frames transmitted");
3971 }
3972 
3973 /*
3974 ** Set flow control using sysctl:
3975 ** Flow control values:
3976 ** 	0 - off
3977 **	1 - rx pause
3978 **	2 - tx pause
3979 **	3 - full
3980 */
3981 static int
3982 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
3983 {
3984 	int error, last;
3985 	struct adapter *adapter = (struct adapter *) arg1;
3986 
3987 	last = adapter->fc;
3988 	error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
3989 	if ((error) || (req->newptr == NULL))
3990 		return (error);
3991 
3992 	/* Don't bother if it's not changed */
3993 	if (adapter->fc == last)
3994 		return (0);
3995 
3996 	switch (adapter->fc) {
3997 		case ixgbe_fc_rx_pause:
3998 		case ixgbe_fc_tx_pause:
3999 		case ixgbe_fc_full:
4000 			adapter->hw.fc.requested_mode = adapter->fc;
4001 			if (adapter->num_queues > 1)
4002 				ixgbe_disable_rx_drop(adapter);
4003 			break;
4004 		case ixgbe_fc_none:
4005 			adapter->hw.fc.requested_mode = ixgbe_fc_none;
4006 			if (adapter->num_queues > 1)
4007 				ixgbe_enable_rx_drop(adapter);
4008 			break;
4009 		default:
4010 			adapter->fc = last;
4011 			return (EINVAL);
4012 	}
4013 	/* Don't autoneg if forcing a value */
4014 	adapter->hw.fc.disable_fc_autoneg = TRUE;
4015 	ixgbe_fc_enable(&adapter->hw);
4016 	return error;
4017 }
4018 
4019 /*
4020 ** Control advertised link speed:
4021 **	Flags:
4022 **	0x1 - advertise 100 Mb
4023 **	0x2 - advertise 1G
4024 **	0x4 - advertise 10G
4025 */
4026 static int
4027 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
4028 {
4029 	int			error = 0, requested;
4030 	struct adapter		*adapter;
4031 	device_t		dev;
4032 	struct ixgbe_hw		*hw;
4033 	ixgbe_link_speed	speed = 0;
4034 
4035 	adapter = (struct adapter *) arg1;
4036 	dev = adapter->dev;
4037 	hw = &adapter->hw;
4038 
4039 	requested = adapter->advertise;
4040 	error = sysctl_handle_int(oidp, &requested, 0, req);
4041 	if ((error) || (req->newptr == NULL))
4042 		return (error);
4043 
4044 	/* Checks to validate new value */
4045 	if (adapter->advertise == requested) /* no change */
4046 		return (0);
4047 
4048 	if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
4049 	    (hw->phy.multispeed_fiber))) {
4050 		device_printf(dev,
4051 		    "Advertised speed can only be set on copper or "
4052 		    "multispeed fiber media types.\n");
4053 		return (EINVAL);
4054 	}
4055 
4056 	if (requested < 0x1 || requested > 0x7) {
4057 		device_printf(dev,
4058 		    "Invalid advertised speed; valid modes are 0x1 through 0x7\n");
4059 		return (EINVAL);
4060 	}
4061 
4062 	if ((requested & 0x1)
4063 	    && (hw->mac.type != ixgbe_mac_X540)
4064 	    && (hw->mac.type != ixgbe_mac_X550)) {
4065 		device_printf(dev, "Set Advertise: 100Mb on X540/X550 only\n");
4066 		return (EINVAL);
4067 	}
4068 
4069 	/* Set new value and report new advertised mode */
4070 	if (requested & 0x1)
4071 		speed |= IXGBE_LINK_SPEED_100_FULL;
4072 	if (requested & 0x2)
4073 		speed |= IXGBE_LINK_SPEED_1GB_FULL;
4074 	if (requested & 0x4)
4075 		speed |= IXGBE_LINK_SPEED_10GB_FULL;
4076 
4077 	hw->mac.autotry_restart = TRUE;
4078 	hw->mac.ops.setup_link(hw, speed, TRUE);
4079 	adapter->advertise = requested;
4080 
4081 	return (error);
4082 }
4083 
4084 /*
4085 ** Thermal Shutdown Trigger
4086 **   - cause a Thermal Overtemp IRQ
4087 **   - this now requires firmware enabling
4088 */
4089 static int
4090 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
4091 {
4092 	int		error, fire = 0;
4093 	struct adapter	*adapter = (struct adapter *) arg1;
4094 	struct ixgbe_hw *hw = &adapter->hw;
4095 
4096 
4097 	if (hw->mac.type < ixgbe_mac_X540)
4098 		return (0);
4099 
4100 	error = sysctl_handle_int(oidp, &fire, 0, req);
4101 	if ((error) || (req->newptr == NULL))
4102 		return (error);
4103 
4104 	if (fire) {
4105 		u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
4106 		reg |= IXGBE_EICR_TS;
4107 		IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
4108 	}
4109 
4110 	return (0);
4111 }
4112 
4113 /*
4114 ** Enable the hardware to drop packets when the buffer is
4115 ** full. This is useful when multiqueue,so that no single
4116 ** queue being full stalls the entire RX engine. We only
4117 ** enable this when Multiqueue AND when Flow Control is
4118 ** disabled.
4119 */
4120 static void
4121 ixgbe_enable_rx_drop(struct adapter *adapter)
4122 {
4123         struct ixgbe_hw *hw = &adapter->hw;
4124 
4125 	for (int i = 0; i < adapter->num_queues; i++) {
4126         	u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4127         	srrctl |= IXGBE_SRRCTL_DROP_EN;
4128         	IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4129 	}
4130 }
4131 
4132 static void
4133 ixgbe_disable_rx_drop(struct adapter *adapter)
4134 {
4135         struct ixgbe_hw *hw = &adapter->hw;
4136 
4137 	for (int i = 0; i < adapter->num_queues; i++) {
4138         	u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4139         	srrctl &= ~IXGBE_SRRCTL_DROP_EN;
4140         	IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4141 	}
4142 }
4143 
4144 static void
4145 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
4146 {
4147 	u32 mask;
4148 
4149 	switch (adapter->hw.mac.type) {
4150 	case ixgbe_mac_82598EB:
4151 		mask = (IXGBE_EIMS_RTX_QUEUE & queues);
4152 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
4153 		break;
4154 	case ixgbe_mac_82599EB:
4155 	case ixgbe_mac_X540:
4156 	case ixgbe_mac_X550:
4157 		mask = (queues & 0xFFFFFFFF);
4158 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
4159 		mask = (queues >> 32);
4160 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
4161 		break;
4162 	default:
4163 		break;
4164 	}
4165 }
4166 
4167 
4168