xref: /titanic_44/usr/src/uts/common/io/ixgbe/ixgbe_main.c (revision f4c46b1eda9212fd32ba197043d52239ef5c0a7f)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
24  */
25 
26 /*
27  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
29  * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
30  */
31 
32 #include "ixgbe_sw.h"
33 
34 static char ixgbe_ident[] = "Intel 10Gb Ethernet";
35 static char ixgbe_version[] = "ixgbe 1.1.7";
36 
37 /*
38  * Local function protoypes
39  */
40 static int ixgbe_register_mac(ixgbe_t *);
41 static int ixgbe_identify_hardware(ixgbe_t *);
42 static int ixgbe_regs_map(ixgbe_t *);
43 static void ixgbe_init_properties(ixgbe_t *);
44 static int ixgbe_init_driver_settings(ixgbe_t *);
45 static void ixgbe_init_locks(ixgbe_t *);
46 static void ixgbe_destroy_locks(ixgbe_t *);
47 static int ixgbe_init(ixgbe_t *);
48 static int ixgbe_chip_start(ixgbe_t *);
49 static void ixgbe_chip_stop(ixgbe_t *);
50 static int ixgbe_reset(ixgbe_t *);
51 static void ixgbe_tx_clean(ixgbe_t *);
52 static boolean_t ixgbe_tx_drain(ixgbe_t *);
53 static boolean_t ixgbe_rx_drain(ixgbe_t *);
54 static int ixgbe_alloc_rings(ixgbe_t *);
55 static void ixgbe_free_rings(ixgbe_t *);
56 static int ixgbe_alloc_rx_data(ixgbe_t *);
57 static void ixgbe_free_rx_data(ixgbe_t *);
58 static void ixgbe_setup_rings(ixgbe_t *);
59 static void ixgbe_setup_rx(ixgbe_t *);
60 static void ixgbe_setup_tx(ixgbe_t *);
61 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *);
62 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *);
63 static void ixgbe_setup_rss(ixgbe_t *);
64 static void ixgbe_setup_vmdq(ixgbe_t *);
65 static void ixgbe_setup_vmdq_rss(ixgbe_t *);
66 static void ixgbe_init_unicst(ixgbe_t *);
67 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *);
68 static void ixgbe_setup_multicst(ixgbe_t *);
69 static void ixgbe_get_hw_state(ixgbe_t *);
70 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe);
71 static void ixgbe_get_conf(ixgbe_t *);
72 static void ixgbe_init_params(ixgbe_t *);
73 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int);
74 static void ixgbe_driver_link_check(ixgbe_t *);
75 static void ixgbe_sfp_check(void *);
76 static void ixgbe_overtemp_check(void *);
77 static void ixgbe_link_timer(void *);
78 static void ixgbe_local_timer(void *);
79 static void ixgbe_arm_watchdog_timer(ixgbe_t *);
80 static void ixgbe_restart_watchdog_timer(ixgbe_t *);
81 static void ixgbe_disable_adapter_interrupts(ixgbe_t *);
82 static void ixgbe_enable_adapter_interrupts(ixgbe_t *);
83 static boolean_t is_valid_mac_addr(uint8_t *);
84 static boolean_t ixgbe_stall_check(ixgbe_t *);
85 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t);
86 static void ixgbe_set_internal_mac_loopback(ixgbe_t *);
87 static boolean_t ixgbe_find_mac_address(ixgbe_t *);
88 static int ixgbe_alloc_intrs(ixgbe_t *);
89 static int ixgbe_alloc_intr_handles(ixgbe_t *, int);
90 static int ixgbe_add_intr_handlers(ixgbe_t *);
91 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int);
92 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int);
93 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t);
94 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t);
95 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t);
96 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index);
97 static int ixgbe_map_intrs_to_vectors(ixgbe_t *);
98 static void ixgbe_setup_adapter_vector(ixgbe_t *);
99 static void ixgbe_rem_intr_handlers(ixgbe_t *);
100 static void ixgbe_rem_intrs(ixgbe_t *);
101 static int ixgbe_enable_intrs(ixgbe_t *);
102 static int ixgbe_disable_intrs(ixgbe_t *);
103 static uint_t ixgbe_intr_legacy(void *, void *);
104 static uint_t ixgbe_intr_msi(void *, void *);
105 static uint_t ixgbe_intr_msix(void *, void *);
106 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *);
107 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *);
108 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t);
109 static void ixgbe_get_driver_control(struct ixgbe_hw *);
110 static int ixgbe_addmac(void *, const uint8_t *);
111 static int ixgbe_remmac(void *, const uint8_t *);
112 static void ixgbe_release_driver_control(struct ixgbe_hw *);
113 
114 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t);
115 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t);
116 static int ixgbe_resume(dev_info_t *);
117 static int ixgbe_suspend(dev_info_t *);
118 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *);
119 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *);
120 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *);
121 static int ixgbe_intr_cb_register(ixgbe_t *);
122 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int);
123 
124 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err,
125     const void *impl_data);
126 static void ixgbe_fm_init(ixgbe_t *);
127 static void ixgbe_fm_fini(ixgbe_t *);
128 
129 char *ixgbe_priv_props[] = {
130 	"_tx_copy_thresh",
131 	"_tx_recycle_thresh",
132 	"_tx_overload_thresh",
133 	"_tx_resched_thresh",
134 	"_rx_copy_thresh",
135 	"_rx_limit_per_intr",
136 	"_intr_throttling",
137 	"_adv_pause_cap",
138 	"_adv_asym_pause_cap",
139 	NULL
140 };
141 
142 #define	IXGBE_MAX_PRIV_PROPS \
143 	(sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t))
144 
145 static struct cb_ops ixgbe_cb_ops = {
146 	nulldev,		/* cb_open */
147 	nulldev,		/* cb_close */
148 	nodev,			/* cb_strategy */
149 	nodev,			/* cb_print */
150 	nodev,			/* cb_dump */
151 	nodev,			/* cb_read */
152 	nodev,			/* cb_write */
153 	nodev,			/* cb_ioctl */
154 	nodev,			/* cb_devmap */
155 	nodev,			/* cb_mmap */
156 	nodev,			/* cb_segmap */
157 	nochpoll,		/* cb_chpoll */
158 	ddi_prop_op,		/* cb_prop_op */
159 	NULL,			/* cb_stream */
160 	D_MP | D_HOTPLUG,	/* cb_flag */
161 	CB_REV,			/* cb_rev */
162 	nodev,			/* cb_aread */
163 	nodev			/* cb_awrite */
164 };
165 
166 static struct dev_ops ixgbe_dev_ops = {
167 	DEVO_REV,		/* devo_rev */
168 	0,			/* devo_refcnt */
169 	NULL,			/* devo_getinfo */
170 	nulldev,		/* devo_identify */
171 	nulldev,		/* devo_probe */
172 	ixgbe_attach,		/* devo_attach */
173 	ixgbe_detach,		/* devo_detach */
174 	nodev,			/* devo_reset */
175 	&ixgbe_cb_ops,		/* devo_cb_ops */
176 	NULL,			/* devo_bus_ops */
177 	ddi_power,		/* devo_power */
178 	ddi_quiesce_not_supported,	/* devo_quiesce */
179 };
180 
181 static struct modldrv ixgbe_modldrv = {
182 	&mod_driverops,		/* Type of module.  This one is a driver */
183 	ixgbe_ident,		/* Discription string */
184 	&ixgbe_dev_ops		/* driver ops */
185 };
186 
187 static struct modlinkage ixgbe_modlinkage = {
188 	MODREV_1, &ixgbe_modldrv, NULL
189 };
190 
191 /*
192  * Access attributes for register mapping
193  */
194 ddi_device_acc_attr_t ixgbe_regs_acc_attr = {
195 	DDI_DEVICE_ATTR_V1,
196 	DDI_STRUCTURE_LE_ACC,
197 	DDI_STRICTORDER_ACC,
198 	DDI_FLAGERR_ACC
199 };
200 
201 /*
202  * Loopback property
203  */
204 static lb_property_t lb_normal = {
205 	normal,	"normal", IXGBE_LB_NONE
206 };
207 
208 static lb_property_t lb_mac = {
209 	internal, "MAC", IXGBE_LB_INTERNAL_MAC
210 };
211 
212 static lb_property_t lb_external = {
213 	external, "External", IXGBE_LB_EXTERNAL
214 };
215 
216 #define	IXGBE_M_CALLBACK_FLAGS \
217 	(MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
218 
219 static mac_callbacks_t ixgbe_m_callbacks = {
220 	IXGBE_M_CALLBACK_FLAGS,
221 	ixgbe_m_stat,
222 	ixgbe_m_start,
223 	ixgbe_m_stop,
224 	ixgbe_m_promisc,
225 	ixgbe_m_multicst,
226 	NULL,
227 	NULL,
228 	NULL,
229 	ixgbe_m_ioctl,
230 	ixgbe_m_getcapab,
231 	NULL,
232 	NULL,
233 	ixgbe_m_setprop,
234 	ixgbe_m_getprop,
235 	ixgbe_m_propinfo
236 };
237 
238 /*
239  * Initialize capabilities of each supported adapter type
240  */
241 static adapter_info_t ixgbe_82598eb_cap = {
242 	64,		/* maximum number of rx queues */
243 	1,		/* minimum number of rx queues */
244 	64,		/* default number of rx queues */
245 	16,		/* maximum number of rx groups */
246 	1,		/* minimum number of rx groups */
247 	1,		/* default number of rx groups */
248 	32,		/* maximum number of tx queues */
249 	1,		/* minimum number of tx queues */
250 	8,		/* default number of tx queues */
251 	16366,		/* maximum MTU size */
252 	0xFFFF,		/* maximum interrupt throttle rate */
253 	0,		/* minimum interrupt throttle rate */
254 	200,		/* default interrupt throttle rate */
255 	18,		/* maximum total msix vectors */
256 	16,		/* maximum number of ring vectors */
257 	2,		/* maximum number of other vectors */
258 	IXGBE_EICR_LSC,	/* "other" interrupt types handled */
259 	0,		/* "other" interrupt types enable mask */
260 	(IXGBE_FLAG_DCA_CAPABLE	/* capability flags */
261 	| IXGBE_FLAG_RSS_CAPABLE
262 	| IXGBE_FLAG_VMDQ_CAPABLE)
263 };
264 
265 static adapter_info_t ixgbe_82599eb_cap = {
266 	128,		/* maximum number of rx queues */
267 	1,		/* minimum number of rx queues */
268 	128,		/* default number of rx queues */
269 	64,		/* maximum number of rx groups */
270 	1,		/* minimum number of rx groups */
271 	1,		/* default number of rx groups */
272 	128,		/* maximum number of tx queues */
273 	1,		/* minimum number of tx queues */
274 	8,		/* default number of tx queues */
275 	15500,		/* maximum MTU size */
276 	0xFF8,		/* maximum interrupt throttle rate */
277 	0,		/* minimum interrupt throttle rate */
278 	200,		/* default interrupt throttle rate */
279 	64,		/* maximum total msix vectors */
280 	16,		/* maximum number of ring vectors */
281 	2,		/* maximum number of other vectors */
282 	(IXGBE_EICR_LSC
283 	| IXGBE_EICR_GPI_SDP1
284 	| IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */
285 
286 	(IXGBE_SDP1_GPIEN
287 	| IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */
288 
289 	(IXGBE_FLAG_DCA_CAPABLE
290 	| IXGBE_FLAG_RSS_CAPABLE
291 	| IXGBE_FLAG_VMDQ_CAPABLE
292 	| IXGBE_FLAG_RSC_CAPABLE
293 	| IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */
294 };
295 
296 static adapter_info_t ixgbe_X540_cap = {
297 	128,		/* maximum number of rx queues */
298 	1,		/* minimum number of rx queues */
299 	128,		/* default number of rx queues */
300 	64,		/* maximum number of rx groups */
301 	1,		/* minimum number of rx groups */
302 	1,		/* default number of rx groups */
303 	128,		/* maximum number of tx queues */
304 	1,		/* minimum number of tx queues */
305 	8,		/* default number of tx queues */
306 	15500,		/* maximum MTU size */
307 	0xFF8,		/* maximum interrupt throttle rate */
308 	0,		/* minimum interrupt throttle rate */
309 	200,		/* default interrupt throttle rate */
310 	64,		/* maximum total msix vectors */
311 	16,		/* maximum number of ring vectors */
312 	2,		/* maximum number of other vectors */
313 	(IXGBE_EICR_LSC
314 	| IXGBE_EICR_GPI_SDP1
315 	| IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */
316 
317 	(IXGBE_SDP1_GPIEN
318 	| IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */
319 
320 	(IXGBE_FLAG_DCA_CAPABLE
321 	| IXGBE_FLAG_RSS_CAPABLE
322 	| IXGBE_FLAG_VMDQ_CAPABLE
323 	| IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
324 };
325 
326 /*
327  * Module Initialization Functions.
328  */
329 
330 int
331 _init(void)
332 {
333 	int status;
334 
335 	mac_init_ops(&ixgbe_dev_ops, MODULE_NAME);
336 
337 	status = mod_install(&ixgbe_modlinkage);
338 
339 	if (status != DDI_SUCCESS) {
340 		mac_fini_ops(&ixgbe_dev_ops);
341 	}
342 
343 	return (status);
344 }
345 
346 int
347 _fini(void)
348 {
349 	int status;
350 
351 	status = mod_remove(&ixgbe_modlinkage);
352 
353 	if (status == DDI_SUCCESS) {
354 		mac_fini_ops(&ixgbe_dev_ops);
355 	}
356 
357 	return (status);
358 }
359 
360 int
361 _info(struct modinfo *modinfop)
362 {
363 	int status;
364 
365 	status = mod_info(&ixgbe_modlinkage, modinfop);
366 
367 	return (status);
368 }
369 
370 /*
371  * ixgbe_attach - Driver attach.
372  *
373  * This function is the device specific initialization entry
374  * point. This entry point is required and must be written.
375  * The DDI_ATTACH command must be provided in the attach entry
376  * point. When attach() is called with cmd set to DDI_ATTACH,
377  * all normal kernel services (such as kmem_alloc(9F)) are
378  * available for use by the driver.
379  *
380  * The attach() function will be called once for each instance
381  * of  the  device  on  the  system with cmd set to DDI_ATTACH.
382  * Until attach() succeeds, the only driver entry points which
383  * may be called are open(9E) and getinfo(9E).
384  */
385 static int
386 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
387 {
388 	ixgbe_t *ixgbe;
389 	struct ixgbe_osdep *osdep;
390 	struct ixgbe_hw *hw;
391 	int instance;
392 	char taskqname[32];
393 
394 	/*
395 	 * Check the command and perform corresponding operations
396 	 */
397 	switch (cmd) {
398 	default:
399 		return (DDI_FAILURE);
400 
401 	case DDI_RESUME:
402 		return (ixgbe_resume(devinfo));
403 
404 	case DDI_ATTACH:
405 		break;
406 	}
407 
408 	/* Get the device instance */
409 	instance = ddi_get_instance(devinfo);
410 
411 	/* Allocate memory for the instance data structure */
412 	ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP);
413 
414 	ixgbe->dip = devinfo;
415 	ixgbe->instance = instance;
416 
417 	hw = &ixgbe->hw;
418 	osdep = &ixgbe->osdep;
419 	hw->back = osdep;
420 	osdep->ixgbe = ixgbe;
421 
422 	/* Attach the instance pointer to the dev_info data structure */
423 	ddi_set_driver_private(devinfo, ixgbe);
424 
425 	/*
426 	 * Initialize for fma support
427 	 */
428 	ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE,
429 	    0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
430 	    DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
431 	ixgbe_fm_init(ixgbe);
432 	ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT;
433 
434 	/*
435 	 * Map PCI config space registers
436 	 */
437 	if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
438 		ixgbe_error(ixgbe, "Failed to map PCI configurations");
439 		goto attach_fail;
440 	}
441 	ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
442 
443 	/*
444 	 * Identify the chipset family
445 	 */
446 	if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) {
447 		ixgbe_error(ixgbe, "Failed to identify hardware");
448 		goto attach_fail;
449 	}
450 
451 	/*
452 	 * Map device registers
453 	 */
454 	if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) {
455 		ixgbe_error(ixgbe, "Failed to map device registers");
456 		goto attach_fail;
457 	}
458 	ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP;
459 
460 	/*
461 	 * Initialize driver parameters
462 	 */
463 	ixgbe_init_properties(ixgbe);
464 	ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS;
465 
466 	/*
467 	 * Register interrupt callback
468 	 */
469 	if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) {
470 		ixgbe_error(ixgbe, "Failed to register interrupt callback");
471 		goto attach_fail;
472 	}
473 
474 	/*
475 	 * Allocate interrupts
476 	 */
477 	if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) {
478 		ixgbe_error(ixgbe, "Failed to allocate interrupts");
479 		goto attach_fail;
480 	}
481 	ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
482 
483 	/*
484 	 * Allocate rx/tx rings based on the ring numbers.
485 	 * The actual numbers of rx/tx rings are decided by the number of
486 	 * allocated interrupt vectors, so we should allocate the rings after
487 	 * interrupts are allocated.
488 	 */
489 	if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) {
490 		ixgbe_error(ixgbe, "Failed to allocate rx and tx rings");
491 		goto attach_fail;
492 	}
493 	ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS;
494 
495 	/*
496 	 * Map rings to interrupt vectors
497 	 */
498 	if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
499 		ixgbe_error(ixgbe, "Failed to map interrupts to vectors");
500 		goto attach_fail;
501 	}
502 
503 	/*
504 	 * Add interrupt handlers
505 	 */
506 	if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
507 		ixgbe_error(ixgbe, "Failed to add interrupt handlers");
508 		goto attach_fail;
509 	}
510 	ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
511 
512 	/*
513 	 * Create a taskq for sfp-change
514 	 */
515 	(void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance);
516 	if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname,
517 	    1, TASKQ_DEFAULTPRI, 0)) == NULL) {
518 		ixgbe_error(ixgbe, "sfp_taskq create failed");
519 		goto attach_fail;
520 	}
521 	ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ;
522 
523 	/*
524 	 * Create a taskq for over-temp
525 	 */
526 	(void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance);
527 	if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname,
528 	    1, TASKQ_DEFAULTPRI, 0)) == NULL) {
529 		ixgbe_error(ixgbe, "overtemp_taskq create failed");
530 		goto attach_fail;
531 	}
532 	ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ;
533 
534 	/*
535 	 * Initialize driver parameters
536 	 */
537 	if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) {
538 		ixgbe_error(ixgbe, "Failed to initialize driver settings");
539 		goto attach_fail;
540 	}
541 
542 	/*
543 	 * Initialize mutexes for this device.
544 	 * Do this before enabling the interrupt handler and
545 	 * register the softint to avoid the condition where
546 	 * interrupt handler can try using uninitialized mutex.
547 	 */
548 	ixgbe_init_locks(ixgbe);
549 	ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS;
550 
551 	/*
552 	 * Initialize chipset hardware
553 	 */
554 	if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) {
555 		ixgbe_error(ixgbe, "Failed to initialize adapter");
556 		goto attach_fail;
557 	}
558 	ixgbe->link_check_complete = B_FALSE;
559 	ixgbe->link_check_hrtime = gethrtime() +
560 	    (IXGBE_LINK_UP_TIME * 100000000ULL);
561 	ixgbe->attach_progress |= ATTACH_PROGRESS_INIT;
562 
563 	if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) {
564 		ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
565 		goto attach_fail;
566 	}
567 
568 	/*
569 	 * Initialize statistics
570 	 */
571 	if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) {
572 		ixgbe_error(ixgbe, "Failed to initialize statistics");
573 		goto attach_fail;
574 	}
575 	ixgbe->attach_progress |= ATTACH_PROGRESS_STATS;
576 
577 	/*
578 	 * Register the driver to the MAC
579 	 */
580 	if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) {
581 		ixgbe_error(ixgbe, "Failed to register MAC");
582 		goto attach_fail;
583 	}
584 	mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
585 	ixgbe->attach_progress |= ATTACH_PROGRESS_MAC;
586 
587 	ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe,
588 	    IXGBE_CYCLIC_PERIOD, DDI_IPL_0);
589 	if (ixgbe->periodic_id == 0) {
590 		ixgbe_error(ixgbe, "Failed to add the link check timer");
591 		goto attach_fail;
592 	}
593 	ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER;
594 
595 	/*
596 	 * Now that mutex locks are initialized, and the chip is also
597 	 * initialized, enable interrupts.
598 	 */
599 	if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
600 		ixgbe_error(ixgbe, "Failed to enable DDI interrupts");
601 		goto attach_fail;
602 	}
603 	ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
604 
605 	ixgbe_log(ixgbe, "%s, %s", ixgbe_ident, ixgbe_version);
606 	atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED);
607 
608 	return (DDI_SUCCESS);
609 
610 attach_fail:
611 	ixgbe_unconfigure(devinfo, ixgbe);
612 	return (DDI_FAILURE);
613 }
614 
615 /*
616  * ixgbe_detach - Driver detach.
617  *
618  * The detach() function is the complement of the attach routine.
619  * If cmd is set to DDI_DETACH, detach() is used to remove  the
620  * state  associated  with  a  given  instance of a device node
621  * prior to the removal of that instance from the system.
622  *
623  * The detach() function will be called once for each  instance
624  * of the device for which there has been a successful attach()
625  * once there are no longer  any  opens  on  the  device.
626  *
627  * Interrupts routine are disabled, All memory allocated by this
628  * driver are freed.
629  */
630 static int
631 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
632 {
633 	ixgbe_t *ixgbe;
634 
635 	/*
636 	 * Check detach command
637 	 */
638 	switch (cmd) {
639 	default:
640 		return (DDI_FAILURE);
641 
642 	case DDI_SUSPEND:
643 		return (ixgbe_suspend(devinfo));
644 
645 	case DDI_DETACH:
646 		break;
647 	}
648 
649 	/*
650 	 * Get the pointer to the driver private data structure
651 	 */
652 	ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
653 	if (ixgbe == NULL)
654 		return (DDI_FAILURE);
655 
656 	/*
657 	 * If the device is still running, it needs to be stopped first.
658 	 * This check is necessary because under some specific circumstances,
659 	 * the detach routine can be called without stopping the interface
660 	 * first.
661 	 */
662 	if (ixgbe->ixgbe_state & IXGBE_STARTED) {
663 		atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
664 		mutex_enter(&ixgbe->gen_lock);
665 		ixgbe_stop(ixgbe, B_TRUE);
666 		mutex_exit(&ixgbe->gen_lock);
667 		/* Disable and stop the watchdog timer */
668 		ixgbe_disable_watchdog_timer(ixgbe);
669 	}
670 
671 	/*
672 	 * Check if there are still rx buffers held by the upper layer.
673 	 * If so, fail the detach.
674 	 */
675 	if (!ixgbe_rx_drain(ixgbe))
676 		return (DDI_FAILURE);
677 
678 	/*
679 	 * Do the remaining unconfigure routines
680 	 */
681 	ixgbe_unconfigure(devinfo, ixgbe);
682 
683 	return (DDI_SUCCESS);
684 }
685 
686 static void
687 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe)
688 {
689 	/*
690 	 * Disable interrupt
691 	 */
692 	if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
693 		(void) ixgbe_disable_intrs(ixgbe);
694 	}
695 
696 	/*
697 	 * remove the link check timer
698 	 */
699 	if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) {
700 		if (ixgbe->periodic_id != NULL) {
701 			ddi_periodic_delete(ixgbe->periodic_id);
702 			ixgbe->periodic_id = NULL;
703 		}
704 	}
705 
706 	/*
707 	 * Unregister MAC
708 	 */
709 	if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) {
710 		(void) mac_unregister(ixgbe->mac_hdl);
711 	}
712 
713 	/*
714 	 * Free statistics
715 	 */
716 	if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) {
717 		kstat_delete((kstat_t *)ixgbe->ixgbe_ks);
718 	}
719 
720 	/*
721 	 * Remove interrupt handlers
722 	 */
723 	if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
724 		ixgbe_rem_intr_handlers(ixgbe);
725 	}
726 
727 	/*
728 	 * Remove taskq for sfp-status-change
729 	 */
730 	if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) {
731 		ddi_taskq_destroy(ixgbe->sfp_taskq);
732 	}
733 
734 	/*
735 	 * Remove taskq for over-temp
736 	 */
737 	if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) {
738 		ddi_taskq_destroy(ixgbe->overtemp_taskq);
739 	}
740 
741 	/*
742 	 * Remove interrupts
743 	 */
744 	if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
745 		ixgbe_rem_intrs(ixgbe);
746 	}
747 
748 	/*
749 	 * Unregister interrupt callback handler
750 	 */
751 	(void) ddi_cb_unregister(ixgbe->cb_hdl);
752 
753 	/*
754 	 * Remove driver properties
755 	 */
756 	if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) {
757 		(void) ddi_prop_remove_all(devinfo);
758 	}
759 
760 	/*
761 	 * Stop the chipset
762 	 */
763 	if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) {
764 		mutex_enter(&ixgbe->gen_lock);
765 		ixgbe_chip_stop(ixgbe);
766 		mutex_exit(&ixgbe->gen_lock);
767 	}
768 
769 	/*
770 	 * Free register handle
771 	 */
772 	if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) {
773 		if (ixgbe->osdep.reg_handle != NULL)
774 			ddi_regs_map_free(&ixgbe->osdep.reg_handle);
775 	}
776 
777 	/*
778 	 * Free PCI config handle
779 	 */
780 	if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) {
781 		if (ixgbe->osdep.cfg_handle != NULL)
782 			pci_config_teardown(&ixgbe->osdep.cfg_handle);
783 	}
784 
785 	/*
786 	 * Free locks
787 	 */
788 	if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) {
789 		ixgbe_destroy_locks(ixgbe);
790 	}
791 
792 	/*
793 	 * Free the rx/tx rings
794 	 */
795 	if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) {
796 		ixgbe_free_rings(ixgbe);
797 	}
798 
799 	/*
800 	 * Unregister FMA capabilities
801 	 */
802 	if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) {
803 		ixgbe_fm_fini(ixgbe);
804 	}
805 
806 	/*
807 	 * Free the driver data structure
808 	 */
809 	kmem_free(ixgbe, sizeof (ixgbe_t));
810 
811 	ddi_set_driver_private(devinfo, NULL);
812 }
813 
814 /*
815  * ixgbe_register_mac - Register the driver and its function pointers with
816  * the GLD interface.
817  */
818 static int
819 ixgbe_register_mac(ixgbe_t *ixgbe)
820 {
821 	struct ixgbe_hw *hw = &ixgbe->hw;
822 	mac_register_t *mac;
823 	int status;
824 
825 	if ((mac = mac_alloc(MAC_VERSION)) == NULL)
826 		return (IXGBE_FAILURE);
827 
828 	mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
829 	mac->m_driver = ixgbe;
830 	mac->m_dip = ixgbe->dip;
831 	mac->m_src_addr = hw->mac.addr;
832 	mac->m_callbacks = &ixgbe_m_callbacks;
833 	mac->m_min_sdu = 0;
834 	mac->m_max_sdu = ixgbe->default_mtu;
835 	mac->m_margin = VLAN_TAGSZ;
836 	mac->m_priv_props = ixgbe_priv_props;
837 	mac->m_v12n = MAC_VIRT_LEVEL1;
838 
839 	status = mac_register(mac, &ixgbe->mac_hdl);
840 
841 	mac_free(mac);
842 
843 	return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE);
844 }
845 
846 /*
847  * ixgbe_identify_hardware - Identify the type of the chipset.
848  */
849 static int
850 ixgbe_identify_hardware(ixgbe_t *ixgbe)
851 {
852 	struct ixgbe_hw *hw = &ixgbe->hw;
853 	struct ixgbe_osdep *osdep = &ixgbe->osdep;
854 
855 	/*
856 	 * Get the device id
857 	 */
858 	hw->vendor_id =
859 	    pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID);
860 	hw->device_id =
861 	    pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID);
862 	hw->revision_id =
863 	    pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID);
864 	hw->subsystem_device_id =
865 	    pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID);
866 	hw->subsystem_vendor_id =
867 	    pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID);
868 
869 	/*
870 	 * Set the mac type of the adapter based on the device id
871 	 */
872 	if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) {
873 		return (IXGBE_FAILURE);
874 	}
875 
876 	/*
877 	 * Install adapter capabilities
878 	 */
879 	switch (hw->mac.type) {
880 	case ixgbe_mac_82598EB:
881 		IXGBE_DEBUGLOG_0(ixgbe, "identify 82598 adapter\n");
882 		ixgbe->capab = &ixgbe_82598eb_cap;
883 
884 		if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) {
885 			ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
886 			ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1;
887 			ixgbe->capab->other_gpie |= IXGBE_SDP1_GPIEN;
888 		}
889 		break;
890 
891 	case ixgbe_mac_82599EB:
892 		IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n");
893 		ixgbe->capab = &ixgbe_82599eb_cap;
894 
895 		if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) {
896 			ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE;
897 			ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0;
898 			ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN;
899 		}
900 		break;
901 
902 	case ixgbe_mac_X540:
903 		IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n");
904 		ixgbe->capab = &ixgbe_X540_cap;
905 		/*
906 		 * For now, X540 is all set in its capab structure.
907 		 * As other X540 variants show up, things can change here.
908 		 */
909 		break;
910 
911 	default:
912 		IXGBE_DEBUGLOG_1(ixgbe,
913 		    "adapter not supported in ixgbe_identify_hardware(): %d\n",
914 		    hw->mac.type);
915 		return (IXGBE_FAILURE);
916 	}
917 
918 	return (IXGBE_SUCCESS);
919 }
920 
921 /*
922  * ixgbe_regs_map - Map the device registers.
923  *
924  */
925 static int
926 ixgbe_regs_map(ixgbe_t *ixgbe)
927 {
928 	dev_info_t *devinfo = ixgbe->dip;
929 	struct ixgbe_hw *hw = &ixgbe->hw;
930 	struct ixgbe_osdep *osdep = &ixgbe->osdep;
931 	off_t mem_size;
932 
933 	/*
934 	 * First get the size of device registers to be mapped.
935 	 */
936 	if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size)
937 	    != DDI_SUCCESS) {
938 		return (IXGBE_FAILURE);
939 	}
940 
941 	/*
942 	 * Call ddi_regs_map_setup() to map registers
943 	 */
944 	if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET,
945 	    (caddr_t *)&hw->hw_addr, 0,
946 	    mem_size, &ixgbe_regs_acc_attr,
947 	    &osdep->reg_handle)) != DDI_SUCCESS) {
948 		return (IXGBE_FAILURE);
949 	}
950 
951 	return (IXGBE_SUCCESS);
952 }
953 
954 /*
955  * ixgbe_init_properties - Initialize driver properties.
956  */
957 static void
958 ixgbe_init_properties(ixgbe_t *ixgbe)
959 {
960 	/*
961 	 * Get conf file properties, including link settings
962 	 * jumbo frames, ring number, descriptor number, etc.
963 	 */
964 	ixgbe_get_conf(ixgbe);
965 
966 	ixgbe_init_params(ixgbe);
967 }
968 
969 /*
970  * ixgbe_init_driver_settings - Initialize driver settings.
971  *
972  * The settings include hardware function pointers, bus information,
973  * rx/tx rings settings, link state, and any other parameters that
974  * need to be setup during driver initialization.
975  */
976 static int
977 ixgbe_init_driver_settings(ixgbe_t *ixgbe)
978 {
979 	struct ixgbe_hw *hw = &ixgbe->hw;
980 	dev_info_t *devinfo = ixgbe->dip;
981 	ixgbe_rx_ring_t *rx_ring;
982 	ixgbe_rx_group_t *rx_group;
983 	ixgbe_tx_ring_t *tx_ring;
984 	uint32_t rx_size;
985 	uint32_t tx_size;
986 	uint32_t ring_per_group;
987 	int i;
988 
989 	/*
990 	 * Initialize chipset specific hardware function pointers
991 	 */
992 	if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) {
993 		return (IXGBE_FAILURE);
994 	}
995 
996 	/*
997 	 * Get the system page size
998 	 */
999 	ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1);
1000 
1001 	/*
1002 	 * Set rx buffer size
1003 	 *
1004 	 * The IP header alignment room is counted in the calculation.
1005 	 * The rx buffer size is in unit of 1K that is required by the
1006 	 * chipset hardware.
1007 	 */
1008 	rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM;
1009 	ixgbe->rx_buf_size = ((rx_size >> 10) +
1010 	    ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1011 
1012 	/*
1013 	 * Set tx buffer size
1014 	 */
1015 	tx_size = ixgbe->max_frame_size;
1016 	ixgbe->tx_buf_size = ((tx_size >> 10) +
1017 	    ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1018 
1019 	/*
1020 	 * Initialize rx/tx rings/groups parameters
1021 	 */
1022 	ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
1023 	for (i = 0; i < ixgbe->num_rx_rings; i++) {
1024 		rx_ring = &ixgbe->rx_rings[i];
1025 		rx_ring->index = i;
1026 		rx_ring->ixgbe = ixgbe;
1027 		rx_ring->group_index = i / ring_per_group;
1028 		rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i);
1029 	}
1030 
1031 	for (i = 0; i < ixgbe->num_rx_groups; i++) {
1032 		rx_group = &ixgbe->rx_groups[i];
1033 		rx_group->index = i;
1034 		rx_group->ixgbe = ixgbe;
1035 	}
1036 
1037 	for (i = 0; i < ixgbe->num_tx_rings; i++) {
1038 		tx_ring = &ixgbe->tx_rings[i];
1039 		tx_ring->index = i;
1040 		tx_ring->ixgbe = ixgbe;
1041 		if (ixgbe->tx_head_wb_enable)
1042 			tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb;
1043 		else
1044 			tx_ring->tx_recycle = ixgbe_tx_recycle_legacy;
1045 
1046 		tx_ring->ring_size = ixgbe->tx_ring_size;
1047 		tx_ring->free_list_size = ixgbe->tx_ring_size +
1048 		    (ixgbe->tx_ring_size >> 1);
1049 	}
1050 
1051 	/*
1052 	 * Initialize values of interrupt throttling rate
1053 	 */
1054 	for (i = 1; i < MAX_INTR_VECTOR; i++)
1055 		ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0];
1056 
1057 	/*
1058 	 * The initial link state should be "unknown"
1059 	 */
1060 	ixgbe->link_state = LINK_STATE_UNKNOWN;
1061 
1062 	return (IXGBE_SUCCESS);
1063 }
1064 
1065 /*
1066  * ixgbe_init_locks - Initialize locks.
1067  */
1068 static void
1069 ixgbe_init_locks(ixgbe_t *ixgbe)
1070 {
1071 	ixgbe_rx_ring_t *rx_ring;
1072 	ixgbe_tx_ring_t *tx_ring;
1073 	int i;
1074 
1075 	for (i = 0; i < ixgbe->num_rx_rings; i++) {
1076 		rx_ring = &ixgbe->rx_rings[i];
1077 		mutex_init(&rx_ring->rx_lock, NULL,
1078 		    MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1079 	}
1080 
1081 	for (i = 0; i < ixgbe->num_tx_rings; i++) {
1082 		tx_ring = &ixgbe->tx_rings[i];
1083 		mutex_init(&tx_ring->tx_lock, NULL,
1084 		    MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1085 		mutex_init(&tx_ring->recycle_lock, NULL,
1086 		    MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1087 		mutex_init(&tx_ring->tcb_head_lock, NULL,
1088 		    MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1089 		mutex_init(&tx_ring->tcb_tail_lock, NULL,
1090 		    MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1091 	}
1092 
1093 	mutex_init(&ixgbe->gen_lock, NULL,
1094 	    MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1095 
1096 	mutex_init(&ixgbe->watchdog_lock, NULL,
1097 	    MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1098 }
1099 
1100 /*
1101  * ixgbe_destroy_locks - Destroy locks.
1102  */
1103 static void
1104 ixgbe_destroy_locks(ixgbe_t *ixgbe)
1105 {
1106 	ixgbe_rx_ring_t *rx_ring;
1107 	ixgbe_tx_ring_t *tx_ring;
1108 	int i;
1109 
1110 	for (i = 0; i < ixgbe->num_rx_rings; i++) {
1111 		rx_ring = &ixgbe->rx_rings[i];
1112 		mutex_destroy(&rx_ring->rx_lock);
1113 	}
1114 
1115 	for (i = 0; i < ixgbe->num_tx_rings; i++) {
1116 		tx_ring = &ixgbe->tx_rings[i];
1117 		mutex_destroy(&tx_ring->tx_lock);
1118 		mutex_destroy(&tx_ring->recycle_lock);
1119 		mutex_destroy(&tx_ring->tcb_head_lock);
1120 		mutex_destroy(&tx_ring->tcb_tail_lock);
1121 	}
1122 
1123 	mutex_destroy(&ixgbe->gen_lock);
1124 	mutex_destroy(&ixgbe->watchdog_lock);
1125 }
1126 
1127 static int
1128 ixgbe_resume(dev_info_t *devinfo)
1129 {
1130 	ixgbe_t *ixgbe;
1131 	int i;
1132 
1133 	ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1134 	if (ixgbe == NULL)
1135 		return (DDI_FAILURE);
1136 
1137 	mutex_enter(&ixgbe->gen_lock);
1138 
1139 	if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1140 		if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1141 			mutex_exit(&ixgbe->gen_lock);
1142 			return (DDI_FAILURE);
1143 		}
1144 
1145 		/*
1146 		 * Enable and start the watchdog timer
1147 		 */
1148 		ixgbe_enable_watchdog_timer(ixgbe);
1149 	}
1150 
1151 	atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED);
1152 
1153 	if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1154 		for (i = 0; i < ixgbe->num_tx_rings; i++) {
1155 			mac_tx_ring_update(ixgbe->mac_hdl,
1156 			    ixgbe->tx_rings[i].ring_handle);
1157 		}
1158 	}
1159 
1160 	mutex_exit(&ixgbe->gen_lock);
1161 
1162 	return (DDI_SUCCESS);
1163 }
1164 
1165 static int
1166 ixgbe_suspend(dev_info_t *devinfo)
1167 {
1168 	ixgbe_t *ixgbe;
1169 
1170 	ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1171 	if (ixgbe == NULL)
1172 		return (DDI_FAILURE);
1173 
1174 	mutex_enter(&ixgbe->gen_lock);
1175 
1176 	atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED);
1177 	if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1178 		mutex_exit(&ixgbe->gen_lock);
1179 		return (DDI_SUCCESS);
1180 	}
1181 	ixgbe_stop(ixgbe, B_FALSE);
1182 
1183 	mutex_exit(&ixgbe->gen_lock);
1184 
1185 	/*
1186 	 * Disable and stop the watchdog timer
1187 	 */
1188 	ixgbe_disable_watchdog_timer(ixgbe);
1189 
1190 	return (DDI_SUCCESS);
1191 }
1192 
1193 /*
1194  * ixgbe_init - Initialize the device.
1195  */
1196 static int
1197 ixgbe_init(ixgbe_t *ixgbe)
1198 {
1199 	struct ixgbe_hw *hw = &ixgbe->hw;
1200 
1201 	mutex_enter(&ixgbe->gen_lock);
1202 
1203 	/*
1204 	 * Reset chipset to put the hardware in a known state
1205 	 * before we try to do anything with the eeprom.
1206 	 */
1207 	if (ixgbe_reset_hw(hw) != IXGBE_SUCCESS) {
1208 		ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1209 		goto init_fail;
1210 	}
1211 
1212 	/*
1213 	 * Need to init eeprom before validating the checksum.
1214 	 */
1215 	if (ixgbe_init_eeprom_params(hw) < 0) {
1216 		ixgbe_error(ixgbe,
1217 		    "Unable to intitialize the eeprom interface.");
1218 		ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1219 		goto init_fail;
1220 	}
1221 
1222 	/*
1223 	 * NVM validation
1224 	 */
1225 	if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1226 		/*
1227 		 * Some PCI-E parts fail the first check due to
1228 		 * the link being in sleep state.  Call it again,
1229 		 * if it fails a second time it's a real issue.
1230 		 */
1231 		if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1232 			ixgbe_error(ixgbe,
1233 			    "Invalid NVM checksum. Please contact "
1234 			    "the vendor to update the NVM.");
1235 			ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1236 			goto init_fail;
1237 		}
1238 	}
1239 
1240 	/*
1241 	 * Setup default flow control thresholds - enable/disable
1242 	 * & flow control type is controlled by ixgbe.conf
1243 	 */
1244 	hw->fc.high_water[0] = DEFAULT_FCRTH;
1245 	hw->fc.low_water[0] = DEFAULT_FCRTL;
1246 	hw->fc.pause_time = DEFAULT_FCPAUSE;
1247 	hw->fc.send_xon = B_TRUE;
1248 
1249 	/*
1250 	 * Initialize link settings
1251 	 */
1252 	(void) ixgbe_driver_setup_link(ixgbe, B_FALSE);
1253 
1254 	/*
1255 	 * Initialize the chipset hardware
1256 	 */
1257 	if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1258 		ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1259 		goto init_fail;
1260 	}
1261 
1262 	if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1263 		goto init_fail;
1264 	}
1265 
1266 	mutex_exit(&ixgbe->gen_lock);
1267 	return (IXGBE_SUCCESS);
1268 
1269 init_fail:
1270 	/*
1271 	 * Reset PHY
1272 	 */
1273 	(void) ixgbe_reset_phy(hw);
1274 
1275 	mutex_exit(&ixgbe->gen_lock);
1276 	ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1277 	return (IXGBE_FAILURE);
1278 }
1279 
1280 /*
1281  * ixgbe_chip_start - Initialize and start the chipset hardware.
1282  */
1283 static int
1284 ixgbe_chip_start(ixgbe_t *ixgbe)
1285 {
1286 	struct ixgbe_hw *hw = &ixgbe->hw;
1287 	int ret_val, i;
1288 
1289 	ASSERT(mutex_owned(&ixgbe->gen_lock));
1290 
1291 	/*
1292 	 * Get the mac address
1293 	 * This function should handle SPARC case correctly.
1294 	 */
1295 	if (!ixgbe_find_mac_address(ixgbe)) {
1296 		ixgbe_error(ixgbe, "Failed to get the mac address");
1297 		return (IXGBE_FAILURE);
1298 	}
1299 
1300 	/*
1301 	 * Validate the mac address
1302 	 */
1303 	(void) ixgbe_init_rx_addrs(hw);
1304 	if (!is_valid_mac_addr(hw->mac.addr)) {
1305 		ixgbe_error(ixgbe, "Invalid mac address");
1306 		return (IXGBE_FAILURE);
1307 	}
1308 
1309 	/*
1310 	 * Configure/Initialize hardware
1311 	 */
1312 	ret_val = ixgbe_init_hw(hw);
1313 	if (ret_val != IXGBE_SUCCESS) {
1314 		if (ret_val == IXGBE_ERR_EEPROM_VERSION) {
1315 			ixgbe_error(ixgbe,
1316 			    "This 82599 device is pre-release and contains"
1317 			    " outdated firmware, please contact your hardware"
1318 			    " vendor for a replacement.");
1319 		} else {
1320 			ixgbe_error(ixgbe, "Failed to initialize hardware");
1321 			return (IXGBE_FAILURE);
1322 		}
1323 	}
1324 
1325 	/*
1326 	 * Re-enable relaxed ordering for performance.  It is disabled
1327 	 * by default in the hardware init.
1328 	 */
1329 	if (ixgbe->relax_order_enable == B_TRUE)
1330 		ixgbe_enable_relaxed_ordering(hw);
1331 
1332 	/*
1333 	 * Setup adapter interrupt vectors
1334 	 */
1335 	ixgbe_setup_adapter_vector(ixgbe);
1336 
1337 	/*
1338 	 * Initialize unicast addresses.
1339 	 */
1340 	ixgbe_init_unicst(ixgbe);
1341 
1342 	/*
1343 	 * Setup and initialize the mctable structures.
1344 	 */
1345 	ixgbe_setup_multicst(ixgbe);
1346 
1347 	/*
1348 	 * Set interrupt throttling rate
1349 	 */
1350 	for (i = 0; i < ixgbe->intr_cnt; i++) {
1351 		IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]);
1352 	}
1353 
1354 	/*
1355 	 * Save the state of the phy
1356 	 */
1357 	ixgbe_get_hw_state(ixgbe);
1358 
1359 	/*
1360 	 * Make sure driver has control
1361 	 */
1362 	ixgbe_get_driver_control(hw);
1363 
1364 	return (IXGBE_SUCCESS);
1365 }
1366 
1367 /*
1368  * ixgbe_chip_stop - Stop the chipset hardware
1369  */
1370 static void
1371 ixgbe_chip_stop(ixgbe_t *ixgbe)
1372 {
1373 	struct ixgbe_hw *hw = &ixgbe->hw;
1374 
1375 	ASSERT(mutex_owned(&ixgbe->gen_lock));
1376 
1377 	/*
1378 	 * Tell firmware driver is no longer in control
1379 	 */
1380 	ixgbe_release_driver_control(hw);
1381 
1382 	/*
1383 	 * Reset the chipset
1384 	 */
1385 	(void) ixgbe_reset_hw(hw);
1386 
1387 	/*
1388 	 * Reset PHY
1389 	 */
1390 	(void) ixgbe_reset_phy(hw);
1391 }
1392 
1393 /*
1394  * ixgbe_reset - Reset the chipset and re-start the driver.
1395  *
1396  * It involves stopping and re-starting the chipset,
1397  * and re-configuring the rx/tx rings.
1398  */
1399 static int
1400 ixgbe_reset(ixgbe_t *ixgbe)
1401 {
1402 	int i;
1403 
1404 	/*
1405 	 * Disable and stop the watchdog timer
1406 	 */
1407 	ixgbe_disable_watchdog_timer(ixgbe);
1408 
1409 	mutex_enter(&ixgbe->gen_lock);
1410 
1411 	ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED);
1412 	atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
1413 
1414 	ixgbe_stop(ixgbe, B_FALSE);
1415 
1416 	if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1417 		mutex_exit(&ixgbe->gen_lock);
1418 		return (IXGBE_FAILURE);
1419 	}
1420 
1421 	/*
1422 	 * After resetting, need to recheck the link status.
1423 	 */
1424 	ixgbe->link_check_complete = B_FALSE;
1425 	ixgbe->link_check_hrtime = gethrtime() +
1426 	    (IXGBE_LINK_UP_TIME * 100000000ULL);
1427 
1428 	atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED);
1429 
1430 	if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) {
1431 		for (i = 0; i < ixgbe->num_tx_rings; i++) {
1432 			mac_tx_ring_update(ixgbe->mac_hdl,
1433 			    ixgbe->tx_rings[i].ring_handle);
1434 		}
1435 	}
1436 
1437 	mutex_exit(&ixgbe->gen_lock);
1438 
1439 	/*
1440 	 * Enable and start the watchdog timer
1441 	 */
1442 	ixgbe_enable_watchdog_timer(ixgbe);
1443 
1444 	return (IXGBE_SUCCESS);
1445 }
1446 
1447 /*
1448  * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources.
1449  */
1450 static void
1451 ixgbe_tx_clean(ixgbe_t *ixgbe)
1452 {
1453 	ixgbe_tx_ring_t *tx_ring;
1454 	tx_control_block_t *tcb;
1455 	link_list_t pending_list;
1456 	uint32_t desc_num;
1457 	int i, j;
1458 
1459 	LINK_LIST_INIT(&pending_list);
1460 
1461 	for (i = 0; i < ixgbe->num_tx_rings; i++) {
1462 		tx_ring = &ixgbe->tx_rings[i];
1463 
1464 		mutex_enter(&tx_ring->recycle_lock);
1465 
1466 		/*
1467 		 * Clean the pending tx data - the pending packets in the
1468 		 * work_list that have no chances to be transmitted again.
1469 		 *
1470 		 * We must ensure the chipset is stopped or the link is down
1471 		 * before cleaning the transmit packets.
1472 		 */
1473 		desc_num = 0;
1474 		for (j = 0; j < tx_ring->ring_size; j++) {
1475 			tcb = tx_ring->work_list[j];
1476 			if (tcb != NULL) {
1477 				desc_num += tcb->desc_num;
1478 
1479 				tx_ring->work_list[j] = NULL;
1480 
1481 				ixgbe_free_tcb(tcb);
1482 
1483 				LIST_PUSH_TAIL(&pending_list, &tcb->link);
1484 			}
1485 		}
1486 
1487 		if (desc_num > 0) {
1488 			atomic_add_32(&tx_ring->tbd_free, desc_num);
1489 			ASSERT(tx_ring->tbd_free == tx_ring->ring_size);
1490 
1491 			/*
1492 			 * Reset the head and tail pointers of the tbd ring;
1493 			 * Reset the writeback head if it's enable.
1494 			 */
1495 			tx_ring->tbd_head = 0;
1496 			tx_ring->tbd_tail = 0;
1497 			if (ixgbe->tx_head_wb_enable)
1498 				*tx_ring->tbd_head_wb = 0;
1499 
1500 			IXGBE_WRITE_REG(&ixgbe->hw,
1501 			    IXGBE_TDH(tx_ring->index), 0);
1502 			IXGBE_WRITE_REG(&ixgbe->hw,
1503 			    IXGBE_TDT(tx_ring->index), 0);
1504 		}
1505 
1506 		mutex_exit(&tx_ring->recycle_lock);
1507 
1508 		/*
1509 		 * Add the tx control blocks in the pending list to
1510 		 * the free list.
1511 		 */
1512 		ixgbe_put_free_list(tx_ring, &pending_list);
1513 	}
1514 }
1515 
1516 /*
1517  * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be
1518  * transmitted.
1519  */
1520 static boolean_t
1521 ixgbe_tx_drain(ixgbe_t *ixgbe)
1522 {
1523 	ixgbe_tx_ring_t *tx_ring;
1524 	boolean_t done;
1525 	int i, j;
1526 
1527 	/*
1528 	 * Wait for a specific time to allow pending tx packets
1529 	 * to be transmitted.
1530 	 *
1531 	 * Check the counter tbd_free to see if transmission is done.
1532 	 * No lock protection is needed here.
1533 	 *
1534 	 * Return B_TRUE if all pending packets have been transmitted;
1535 	 * Otherwise return B_FALSE;
1536 	 */
1537 	for (i = 0; i < TX_DRAIN_TIME; i++) {
1538 
1539 		done = B_TRUE;
1540 		for (j = 0; j < ixgbe->num_tx_rings; j++) {
1541 			tx_ring = &ixgbe->tx_rings[j];
1542 			done = done &&
1543 			    (tx_ring->tbd_free == tx_ring->ring_size);
1544 		}
1545 
1546 		if (done)
1547 			break;
1548 
1549 		msec_delay(1);
1550 	}
1551 
1552 	return (done);
1553 }
1554 
1555 /*
1556  * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer.
1557  */
1558 static boolean_t
1559 ixgbe_rx_drain(ixgbe_t *ixgbe)
1560 {
1561 	boolean_t done = B_TRUE;
1562 	int i;
1563 
1564 	/*
1565 	 * Polling the rx free list to check if those rx buffers held by
1566 	 * the upper layer are released.
1567 	 *
1568 	 * Check the counter rcb_free to see if all pending buffers are
1569 	 * released. No lock protection is needed here.
1570 	 *
1571 	 * Return B_TRUE if all pending buffers have been released;
1572 	 * Otherwise return B_FALSE;
1573 	 */
1574 	for (i = 0; i < RX_DRAIN_TIME; i++) {
1575 		done = (ixgbe->rcb_pending == 0);
1576 
1577 		if (done)
1578 			break;
1579 
1580 		msec_delay(1);
1581 	}
1582 
1583 	return (done);
1584 }
1585 
1586 /*
1587  * ixgbe_start - Start the driver/chipset.
1588  */
1589 int
1590 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer)
1591 {
1592 	int i;
1593 
1594 	ASSERT(mutex_owned(&ixgbe->gen_lock));
1595 
1596 	if (alloc_buffer) {
1597 		if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) {
1598 			ixgbe_error(ixgbe,
1599 			    "Failed to allocate software receive rings");
1600 			return (IXGBE_FAILURE);
1601 		}
1602 
1603 		/* Allocate buffers for all the rx/tx rings */
1604 		if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) {
1605 			ixgbe_error(ixgbe, "Failed to allocate DMA resource");
1606 			return (IXGBE_FAILURE);
1607 		}
1608 
1609 		ixgbe->tx_ring_init = B_TRUE;
1610 	} else {
1611 		ixgbe->tx_ring_init = B_FALSE;
1612 	}
1613 
1614 	for (i = 0; i < ixgbe->num_rx_rings; i++)
1615 		mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1616 	for (i = 0; i < ixgbe->num_tx_rings; i++)
1617 		mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1618 
1619 	/*
1620 	 * Start the chipset hardware
1621 	 */
1622 	if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1623 		ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1624 		goto start_failure;
1625 	}
1626 
1627 	if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1628 		goto start_failure;
1629 	}
1630 
1631 	/*
1632 	 * Setup the rx/tx rings
1633 	 */
1634 	ixgbe_setup_rings(ixgbe);
1635 
1636 	/*
1637 	 * ixgbe_start() will be called when resetting, however if reset
1638 	 * happens, we need to clear the ERROR, STALL and OVERTEMP flags
1639 	 * before enabling the interrupts.
1640 	 */
1641 	atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR
1642 	    | IXGBE_STALL| IXGBE_OVERTEMP));
1643 
1644 	/*
1645 	 * Enable adapter interrupts
1646 	 * The interrupts must be enabled after the driver state is START
1647 	 */
1648 	ixgbe_enable_adapter_interrupts(ixgbe);
1649 
1650 	for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1651 		mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1652 	for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1653 		mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1654 
1655 	return (IXGBE_SUCCESS);
1656 
1657 start_failure:
1658 	for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1659 		mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1660 	for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1661 		mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1662 
1663 	ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1664 
1665 	return (IXGBE_FAILURE);
1666 }
1667 
1668 /*
1669  * ixgbe_stop - Stop the driver/chipset.
1670  */
1671 void
1672 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer)
1673 {
1674 	int i;
1675 
1676 	ASSERT(mutex_owned(&ixgbe->gen_lock));
1677 
1678 	/*
1679 	 * Disable the adapter interrupts
1680 	 */
1681 	ixgbe_disable_adapter_interrupts(ixgbe);
1682 
1683 	/*
1684 	 * Drain the pending tx packets
1685 	 */
1686 	(void) ixgbe_tx_drain(ixgbe);
1687 
1688 	for (i = 0; i < ixgbe->num_rx_rings; i++)
1689 		mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1690 	for (i = 0; i < ixgbe->num_tx_rings; i++)
1691 		mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1692 
1693 	/*
1694 	 * Stop the chipset hardware
1695 	 */
1696 	ixgbe_chip_stop(ixgbe);
1697 
1698 	if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1699 		ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1700 	}
1701 
1702 	/*
1703 	 * Clean the pending tx data/resources
1704 	 */
1705 	ixgbe_tx_clean(ixgbe);
1706 
1707 	for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1708 		mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1709 	for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1710 		mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1711 
1712 	if (ixgbe->link_state == LINK_STATE_UP) {
1713 		ixgbe->link_state = LINK_STATE_UNKNOWN;
1714 		mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
1715 	}
1716 
1717 	if (free_buffer) {
1718 		/*
1719 		 * Release the DMA/memory resources of rx/tx rings
1720 		 */
1721 		ixgbe_free_dma(ixgbe);
1722 		ixgbe_free_rx_data(ixgbe);
1723 	}
1724 }
1725 
1726 /*
1727  * ixgbe_cbfunc - Driver interface for generic DDI callbacks
1728  */
1729 /* ARGSUSED */
1730 static int
1731 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg,
1732     void *arg1, void *arg2)
1733 {
1734 	ixgbe_t *ixgbe = (ixgbe_t *)arg1;
1735 
1736 	switch (cbaction) {
1737 	/* IRM callback */
1738 	int count;
1739 	case DDI_CB_INTR_ADD:
1740 	case DDI_CB_INTR_REMOVE:
1741 		count = (int)(uintptr_t)cbarg;
1742 		ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX);
1743 		DTRACE_PROBE2(ixgbe__irm__callback, int, count,
1744 		    int, ixgbe->intr_cnt);
1745 		if (ixgbe_intr_adjust(ixgbe, cbaction, count) !=
1746 		    DDI_SUCCESS) {
1747 			ixgbe_error(ixgbe,
1748 			    "IRM CB: Failed to adjust interrupts");
1749 			goto cb_fail;
1750 		}
1751 		break;
1752 	default:
1753 		IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported",
1754 		    cbaction);
1755 		return (DDI_ENOTSUP);
1756 	}
1757 	return (DDI_SUCCESS);
1758 cb_fail:
1759 	return (DDI_FAILURE);
1760 }
1761 
1762 /*
1763  * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request.
1764  */
1765 static int
1766 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count)
1767 {
1768 	int i, rc, actual;
1769 
1770 	if (count == 0)
1771 		return (DDI_SUCCESS);
1772 
1773 	if ((cbaction == DDI_CB_INTR_ADD &&
1774 	    ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) ||
1775 	    (cbaction == DDI_CB_INTR_REMOVE &&
1776 	    ixgbe->intr_cnt - count < ixgbe->intr_cnt_min))
1777 		return (DDI_FAILURE);
1778 
1779 	if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1780 		return (DDI_FAILURE);
1781 	}
1782 
1783 	for (i = 0; i < ixgbe->num_rx_rings; i++)
1784 		mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL);
1785 	for (i = 0; i < ixgbe->num_tx_rings; i++)
1786 		mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL);
1787 
1788 	mutex_enter(&ixgbe->gen_lock);
1789 	ixgbe->ixgbe_state &= ~IXGBE_STARTED;
1790 	ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST;
1791 	ixgbe->ixgbe_state |= IXGBE_SUSPENDED;
1792 	mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
1793 
1794 	ixgbe_stop(ixgbe, B_FALSE);
1795 	/*
1796 	 * Disable interrupts
1797 	 */
1798 	if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
1799 		rc = ixgbe_disable_intrs(ixgbe);
1800 		ASSERT(rc == IXGBE_SUCCESS);
1801 	}
1802 	ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR;
1803 
1804 	/*
1805 	 * Remove interrupt handlers
1806 	 */
1807 	if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
1808 		ixgbe_rem_intr_handlers(ixgbe);
1809 	}
1810 	ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR;
1811 
1812 	/*
1813 	 * Clear vect_map
1814 	 */
1815 	bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
1816 	switch (cbaction) {
1817 	case DDI_CB_INTR_ADD:
1818 		rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable,
1819 		    DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual,
1820 		    DDI_INTR_ALLOC_NORMAL);
1821 		if (rc != DDI_SUCCESS || actual != count) {
1822 			ixgbe_log(ixgbe, "Adjust interrupts failed."
1823 			    "return: %d, irm cb size: %d, actual: %d",
1824 			    rc, count, actual);
1825 			goto intr_adjust_fail;
1826 		}
1827 		ixgbe->intr_cnt += count;
1828 		break;
1829 
1830 	case DDI_CB_INTR_REMOVE:
1831 		for (i = ixgbe->intr_cnt - count;
1832 		    i < ixgbe->intr_cnt; i ++) {
1833 			rc = ddi_intr_free(ixgbe->htable[i]);
1834 			ixgbe->htable[i] = NULL;
1835 			if (rc != DDI_SUCCESS) {
1836 				ixgbe_log(ixgbe, "Adjust interrupts failed."
1837 				    "return: %d, irm cb size: %d, actual: %d",
1838 				    rc, count, actual);
1839 				goto intr_adjust_fail;
1840 			}
1841 		}
1842 		ixgbe->intr_cnt -= count;
1843 		break;
1844 	}
1845 
1846 	/*
1847 	 * Get priority for first vector, assume remaining are all the same
1848 	 */
1849 	rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
1850 	if (rc != DDI_SUCCESS) {
1851 		ixgbe_log(ixgbe,
1852 		    "Get interrupt priority failed: %d", rc);
1853 		goto intr_adjust_fail;
1854 	}
1855 	rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
1856 	if (rc != DDI_SUCCESS) {
1857 		ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc);
1858 		goto intr_adjust_fail;
1859 	}
1860 	ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
1861 
1862 	/*
1863 	 * Map rings to interrupt vectors
1864 	 */
1865 	if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
1866 		ixgbe_error(ixgbe,
1867 		    "IRM CB: Failed to map interrupts to vectors");
1868 		goto intr_adjust_fail;
1869 	}
1870 
1871 	/*
1872 	 * Add interrupt handlers
1873 	 */
1874 	if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
1875 		ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers");
1876 		goto intr_adjust_fail;
1877 	}
1878 	ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
1879 
1880 	/*
1881 	 * Now that mutex locks are initialized, and the chip is also
1882 	 * initialized, enable interrupts.
1883 	 */
1884 	if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
1885 		ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts");
1886 		goto intr_adjust_fail;
1887 	}
1888 	ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
1889 	if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1890 		ixgbe_error(ixgbe, "IRM CB: Failed to start");
1891 		goto intr_adjust_fail;
1892 	}
1893 	ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST;
1894 	ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED;
1895 	ixgbe->ixgbe_state |= IXGBE_STARTED;
1896 	mutex_exit(&ixgbe->gen_lock);
1897 
1898 	for (i = 0; i < ixgbe->num_rx_rings; i++) {
1899 		mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle,
1900 		    ixgbe->htable[ixgbe->rx_rings[i].intr_vector]);
1901 	}
1902 	for (i = 0; i < ixgbe->num_tx_rings; i++) {
1903 		mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle,
1904 		    ixgbe->htable[ixgbe->tx_rings[i].intr_vector]);
1905 	}
1906 
1907 	/* Wakeup all Tx rings */
1908 	for (i = 0; i < ixgbe->num_tx_rings; i++) {
1909 		mac_tx_ring_update(ixgbe->mac_hdl,
1910 		    ixgbe->tx_rings[i].ring_handle);
1911 	}
1912 
1913 	IXGBE_DEBUGLOG_3(ixgbe,
1914 	    "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).",
1915 	    ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max);
1916 	return (DDI_SUCCESS);
1917 
1918 intr_adjust_fail:
1919 	ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1920 	mutex_exit(&ixgbe->gen_lock);
1921 	return (DDI_FAILURE);
1922 }
1923 
1924 /*
1925  * ixgbe_intr_cb_register - Register interrupt callback function.
1926  */
1927 static int
1928 ixgbe_intr_cb_register(ixgbe_t *ixgbe)
1929 {
1930 	if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc,
1931 	    ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) {
1932 		return (IXGBE_FAILURE);
1933 	}
1934 	IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered.");
1935 	return (IXGBE_SUCCESS);
1936 }
1937 
1938 /*
1939  * ixgbe_alloc_rings - Allocate memory space for rx/tx rings.
1940  */
1941 static int
1942 ixgbe_alloc_rings(ixgbe_t *ixgbe)
1943 {
1944 	/*
1945 	 * Allocate memory space for rx rings
1946 	 */
1947 	ixgbe->rx_rings = kmem_zalloc(
1948 	    sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings,
1949 	    KM_NOSLEEP);
1950 
1951 	if (ixgbe->rx_rings == NULL) {
1952 		return (IXGBE_FAILURE);
1953 	}
1954 
1955 	/*
1956 	 * Allocate memory space for tx rings
1957 	 */
1958 	ixgbe->tx_rings = kmem_zalloc(
1959 	    sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings,
1960 	    KM_NOSLEEP);
1961 
1962 	if (ixgbe->tx_rings == NULL) {
1963 		kmem_free(ixgbe->rx_rings,
1964 		    sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
1965 		ixgbe->rx_rings = NULL;
1966 		return (IXGBE_FAILURE);
1967 	}
1968 
1969 	/*
1970 	 * Allocate memory space for rx ring groups
1971 	 */
1972 	ixgbe->rx_groups = kmem_zalloc(
1973 	    sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups,
1974 	    KM_NOSLEEP);
1975 
1976 	if (ixgbe->rx_groups == NULL) {
1977 		kmem_free(ixgbe->rx_rings,
1978 		    sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
1979 		kmem_free(ixgbe->tx_rings,
1980 		    sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
1981 		ixgbe->rx_rings = NULL;
1982 		ixgbe->tx_rings = NULL;
1983 		return (IXGBE_FAILURE);
1984 	}
1985 
1986 	return (IXGBE_SUCCESS);
1987 }
1988 
1989 /*
1990  * ixgbe_free_rings - Free the memory space of rx/tx rings.
1991  */
1992 static void
1993 ixgbe_free_rings(ixgbe_t *ixgbe)
1994 {
1995 	if (ixgbe->rx_rings != NULL) {
1996 		kmem_free(ixgbe->rx_rings,
1997 		    sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
1998 		ixgbe->rx_rings = NULL;
1999 	}
2000 
2001 	if (ixgbe->tx_rings != NULL) {
2002 		kmem_free(ixgbe->tx_rings,
2003 		    sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2004 		ixgbe->tx_rings = NULL;
2005 	}
2006 
2007 	if (ixgbe->rx_groups != NULL) {
2008 		kmem_free(ixgbe->rx_groups,
2009 		    sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups);
2010 		ixgbe->rx_groups = NULL;
2011 	}
2012 }
2013 
2014 static int
2015 ixgbe_alloc_rx_data(ixgbe_t *ixgbe)
2016 {
2017 	ixgbe_rx_ring_t *rx_ring;
2018 	int i;
2019 
2020 	for (i = 0; i < ixgbe->num_rx_rings; i++) {
2021 		rx_ring = &ixgbe->rx_rings[i];
2022 		if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS)
2023 			goto alloc_rx_rings_failure;
2024 	}
2025 	return (IXGBE_SUCCESS);
2026 
2027 alloc_rx_rings_failure:
2028 	ixgbe_free_rx_data(ixgbe);
2029 	return (IXGBE_FAILURE);
2030 }
2031 
2032 static void
2033 ixgbe_free_rx_data(ixgbe_t *ixgbe)
2034 {
2035 	ixgbe_rx_ring_t *rx_ring;
2036 	ixgbe_rx_data_t *rx_data;
2037 	int i;
2038 
2039 	for (i = 0; i < ixgbe->num_rx_rings; i++) {
2040 		rx_ring = &ixgbe->rx_rings[i];
2041 
2042 		mutex_enter(&ixgbe->rx_pending_lock);
2043 		rx_data = rx_ring->rx_data;
2044 
2045 		if (rx_data != NULL) {
2046 			rx_data->flag |= IXGBE_RX_STOPPED;
2047 
2048 			if (rx_data->rcb_pending == 0) {
2049 				ixgbe_free_rx_ring_data(rx_data);
2050 				rx_ring->rx_data = NULL;
2051 			}
2052 		}
2053 
2054 		mutex_exit(&ixgbe->rx_pending_lock);
2055 	}
2056 }
2057 
2058 /*
2059  * ixgbe_setup_rings - Setup rx/tx rings.
2060  */
2061 static void
2062 ixgbe_setup_rings(ixgbe_t *ixgbe)
2063 {
2064 	/*
2065 	 * Setup the rx/tx rings, including the following:
2066 	 *
2067 	 * 1. Setup the descriptor ring and the control block buffers;
2068 	 * 2. Initialize necessary registers for receive/transmit;
2069 	 * 3. Initialize software pointers/parameters for receive/transmit;
2070 	 */
2071 	ixgbe_setup_rx(ixgbe);
2072 
2073 	ixgbe_setup_tx(ixgbe);
2074 }
2075 
2076 static void
2077 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring)
2078 {
2079 	ixgbe_t *ixgbe = rx_ring->ixgbe;
2080 	ixgbe_rx_data_t *rx_data = rx_ring->rx_data;
2081 	struct ixgbe_hw *hw = &ixgbe->hw;
2082 	rx_control_block_t *rcb;
2083 	union ixgbe_adv_rx_desc	*rbd;
2084 	uint32_t size;
2085 	uint32_t buf_low;
2086 	uint32_t buf_high;
2087 	uint32_t reg_val;
2088 	int i;
2089 
2090 	ASSERT(mutex_owned(&rx_ring->rx_lock));
2091 	ASSERT(mutex_owned(&ixgbe->gen_lock));
2092 
2093 	for (i = 0; i < ixgbe->rx_ring_size; i++) {
2094 		rcb = rx_data->work_list[i];
2095 		rbd = &rx_data->rbd_ring[i];
2096 
2097 		rbd->read.pkt_addr = rcb->rx_buf.dma_address;
2098 		rbd->read.hdr_addr = NULL;
2099 	}
2100 
2101 	/*
2102 	 * Initialize the length register
2103 	 */
2104 	size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc);
2105 	IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size);
2106 
2107 	/*
2108 	 * Initialize the base address registers
2109 	 */
2110 	buf_low = (uint32_t)rx_data->rbd_area.dma_address;
2111 	buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32);
2112 	IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high);
2113 	IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low);
2114 
2115 	/*
2116 	 * Setup head & tail pointers
2117 	 */
2118 	IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index),
2119 	    rx_data->ring_size - 1);
2120 	IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0);
2121 
2122 	rx_data->rbd_next = 0;
2123 	rx_data->lro_first = 0;
2124 
2125 	/*
2126 	 * Setup the Receive Descriptor Control Register (RXDCTL)
2127 	 * PTHRESH=32 descriptors (half the internal cache)
2128 	 * HTHRESH=0 descriptors (to minimize latency on fetch)
2129 	 * WTHRESH defaults to 1 (writeback each descriptor)
2130 	 */
2131 	reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index));
2132 	reg_val |= IXGBE_RXDCTL_ENABLE;	/* enable queue */
2133 
2134 	/* Not a valid value for 82599 or X540 */
2135 	if (hw->mac.type == ixgbe_mac_82598EB) {
2136 		reg_val |= 0x0020;	/* pthresh */
2137 	}
2138 	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val);
2139 
2140 	if (hw->mac.type == ixgbe_mac_82599EB ||
2141 	    hw->mac.type == ixgbe_mac_X540) {
2142 		reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2143 		reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS);
2144 		IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2145 	}
2146 
2147 	/*
2148 	 * Setup the Split and Replication Receive Control Register.
2149 	 * Set the rx buffer size and the advanced descriptor type.
2150 	 */
2151 	reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) |
2152 	    IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2153 	reg_val |= IXGBE_SRRCTL_DROP_EN;
2154 	IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val);
2155 }
2156 
2157 static void
2158 ixgbe_setup_rx(ixgbe_t *ixgbe)
2159 {
2160 	ixgbe_rx_ring_t *rx_ring;
2161 	struct ixgbe_hw *hw = &ixgbe->hw;
2162 	uint32_t reg_val;
2163 	uint32_t ring_mapping;
2164 	uint32_t i, index;
2165 	uint32_t psrtype_rss_bit;
2166 
2167 	/* PSRTYPE must be configured for 82599 */
2168 	if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ &&
2169 	    ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) {
2170 		reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2171 		    IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2172 		reg_val |= IXGBE_PSRTYPE_L2HDR;
2173 		reg_val |= 0x80000000;
2174 		IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val);
2175 	} else {
2176 		if (ixgbe->num_rx_groups > 32) {
2177 			psrtype_rss_bit = 0x20000000;
2178 		} else {
2179 			psrtype_rss_bit = 0x40000000;
2180 		}
2181 		for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) {
2182 			reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2183 			    IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2184 			reg_val |= IXGBE_PSRTYPE_L2HDR;
2185 			reg_val |= psrtype_rss_bit;
2186 			IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val);
2187 		}
2188 	}
2189 
2190 	/*
2191 	 * Set filter control in FCTRL to accept broadcast packets and do
2192 	 * not pass pause frames to host.  Flow control settings are already
2193 	 * in this register, so preserve them.
2194 	 */
2195 	reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2196 	reg_val |= IXGBE_FCTRL_BAM;	/* broadcast accept mode */
2197 	reg_val |= IXGBE_FCTRL_DPF;	/* discard pause frames */
2198 	IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val);
2199 
2200 	/*
2201 	 * Hardware checksum settings
2202 	 */
2203 	if (ixgbe->rx_hcksum_enable) {
2204 		reg_val = IXGBE_RXCSUM_IPPCSE;	/* IP checksum */
2205 		IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val);
2206 	}
2207 
2208 	/*
2209 	 * Setup VMDq and RSS for multiple receive queues
2210 	 */
2211 	switch (ixgbe->classify_mode) {
2212 	case IXGBE_CLASSIFY_RSS:
2213 		/*
2214 		 * One group, only RSS is needed when more than
2215 		 * one ring enabled.
2216 		 */
2217 		ixgbe_setup_rss(ixgbe);
2218 		break;
2219 
2220 	case IXGBE_CLASSIFY_VMDQ:
2221 		/*
2222 		 * Multiple groups, each group has one ring,
2223 		 * only VMDq is needed.
2224 		 */
2225 		ixgbe_setup_vmdq(ixgbe);
2226 		break;
2227 
2228 	case IXGBE_CLASSIFY_VMDQ_RSS:
2229 		/*
2230 		 * Multiple groups and multiple rings, both
2231 		 * VMDq and RSS are needed.
2232 		 */
2233 		ixgbe_setup_vmdq_rss(ixgbe);
2234 		break;
2235 
2236 	default:
2237 		break;
2238 	}
2239 
2240 	/*
2241 	 * Enable the receive unit.  This must be done after filter
2242 	 * control is set in FCTRL.
2243 	 */
2244 	reg_val = (IXGBE_RXCTRL_RXEN	/* Enable Receive Unit */
2245 	    | IXGBE_RXCTRL_DMBYPS);	/* descriptor monitor bypass */
2246 	IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, reg_val);
2247 
2248 	/*
2249 	 * ixgbe_setup_rx_ring must be called after configuring RXCTRL
2250 	 */
2251 	for (i = 0; i < ixgbe->num_rx_rings; i++) {
2252 		rx_ring = &ixgbe->rx_rings[i];
2253 		ixgbe_setup_rx_ring(rx_ring);
2254 	}
2255 
2256 	/*
2257 	 * Setup the per-ring statistics mapping.
2258 	 */
2259 	ring_mapping = 0;
2260 	for (i = 0; i < ixgbe->num_rx_rings; i++) {
2261 		index = ixgbe->rx_rings[i].hw_index;
2262 		ring_mapping = IXGBE_READ_REG(hw, IXGBE_RQSMR(index >> 2));
2263 		ring_mapping |= (i & 0xF) << (8 * (index & 0x3));
2264 		IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), ring_mapping);
2265 	}
2266 
2267 	/*
2268 	 * The Max Frame Size in MHADD/MAXFRS will be internally increased
2269 	 * by four bytes if the packet has a VLAN field, so includes MTU,
2270 	 * ethernet header and frame check sequence.
2271 	 * Register is MAXFRS in 82599.
2272 	 */
2273 	reg_val = (ixgbe->default_mtu + sizeof (struct ether_header)
2274 	    + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT;
2275 	IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val);
2276 
2277 	/*
2278 	 * Setup Jumbo Frame enable bit
2279 	 */
2280 	if (ixgbe->default_mtu > ETHERMTU) {
2281 		reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2282 		reg_val |= IXGBE_HLREG0_JUMBOEN;
2283 		IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2284 	}
2285 
2286 	/*
2287 	 * Setup RSC for multiple receive queues.
2288 	 */
2289 	if (ixgbe->lro_enable) {
2290 		for (i = 0; i < ixgbe->num_rx_rings; i++) {
2291 			/*
2292 			 * Make sure rx_buf_size * MAXDESC not greater
2293 			 * than 65535.
2294 			 * Intel recommends 4 for MAXDESC field value.
2295 			 */
2296 			reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i));
2297 			reg_val |= IXGBE_RSCCTL_RSCEN;
2298 			if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k)
2299 				reg_val |= IXGBE_RSCCTL_MAXDESC_1;
2300 			else
2301 				reg_val |= IXGBE_RSCCTL_MAXDESC_4;
2302 			IXGBE_WRITE_REG(hw,  IXGBE_RSCCTL(i), reg_val);
2303 		}
2304 
2305 		reg_val = IXGBE_READ_REG(hw, IXGBE_RSCDBU);
2306 		reg_val |= IXGBE_RSCDBU_RSCACKDIS;
2307 		IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, reg_val);
2308 
2309 		reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2310 		reg_val |= IXGBE_RDRXCTL_RSCACKC;
2311 		reg_val |= IXGBE_RDRXCTL_FCOE_WRFIX;
2312 		reg_val &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
2313 
2314 		IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2315 	}
2316 }
2317 
2318 static void
2319 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring)
2320 {
2321 	ixgbe_t *ixgbe = tx_ring->ixgbe;
2322 	struct ixgbe_hw *hw = &ixgbe->hw;
2323 	uint32_t size;
2324 	uint32_t buf_low;
2325 	uint32_t buf_high;
2326 	uint32_t reg_val;
2327 
2328 	ASSERT(mutex_owned(&tx_ring->tx_lock));
2329 	ASSERT(mutex_owned(&ixgbe->gen_lock));
2330 
2331 	/*
2332 	 * Initialize the length register
2333 	 */
2334 	size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc);
2335 	IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size);
2336 
2337 	/*
2338 	 * Initialize the base address registers
2339 	 */
2340 	buf_low = (uint32_t)tx_ring->tbd_area.dma_address;
2341 	buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32);
2342 	IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low);
2343 	IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high);
2344 
2345 	/*
2346 	 * Setup head & tail pointers
2347 	 */
2348 	IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0);
2349 	IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0);
2350 
2351 	/*
2352 	 * Setup head write-back
2353 	 */
2354 	if (ixgbe->tx_head_wb_enable) {
2355 		/*
2356 		 * The memory of the head write-back is allocated using
2357 		 * the extra tbd beyond the tail of the tbd ring.
2358 		 */
2359 		tx_ring->tbd_head_wb = (uint32_t *)
2360 		    ((uintptr_t)tx_ring->tbd_area.address + size);
2361 		*tx_ring->tbd_head_wb = 0;
2362 
2363 		buf_low = (uint32_t)
2364 		    (tx_ring->tbd_area.dma_address + size);
2365 		buf_high = (uint32_t)
2366 		    ((tx_ring->tbd_area.dma_address + size) >> 32);
2367 
2368 		/* Set the head write-back enable bit */
2369 		buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE;
2370 
2371 		IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low);
2372 		IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high);
2373 
2374 		/*
2375 		 * Turn off relaxed ordering for head write back or it will
2376 		 * cause problems with the tx recycling
2377 		 */
2378 
2379 		reg_val = (hw->mac.type == ixgbe_mac_82598EB) ?
2380 		    IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) :
2381 		    IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index));
2382 		reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2383 		if (hw->mac.type == ixgbe_mac_82598EB) {
2384 			IXGBE_WRITE_REG(hw,
2385 			    IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
2386 		} else {
2387 			IXGBE_WRITE_REG(hw,
2388 			    IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val);
2389 		}
2390 	} else {
2391 		tx_ring->tbd_head_wb = NULL;
2392 	}
2393 
2394 	tx_ring->tbd_head = 0;
2395 	tx_ring->tbd_tail = 0;
2396 	tx_ring->tbd_free = tx_ring->ring_size;
2397 
2398 	if (ixgbe->tx_ring_init == B_TRUE) {
2399 		tx_ring->tcb_head = 0;
2400 		tx_ring->tcb_tail = 0;
2401 		tx_ring->tcb_free = tx_ring->free_list_size;
2402 	}
2403 
2404 	/*
2405 	 * Initialize the s/w context structure
2406 	 */
2407 	bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t));
2408 }
2409 
2410 static void
2411 ixgbe_setup_tx(ixgbe_t *ixgbe)
2412 {
2413 	struct ixgbe_hw *hw = &ixgbe->hw;
2414 	ixgbe_tx_ring_t *tx_ring;
2415 	uint32_t reg_val;
2416 	uint32_t ring_mapping;
2417 	int i;
2418 
2419 	for (i = 0; i < ixgbe->num_tx_rings; i++) {
2420 		tx_ring = &ixgbe->tx_rings[i];
2421 		ixgbe_setup_tx_ring(tx_ring);
2422 	}
2423 
2424 	/*
2425 	 * Setup the per-ring statistics mapping.
2426 	 */
2427 	ring_mapping = 0;
2428 	for (i = 0; i < ixgbe->num_tx_rings; i++) {
2429 		ring_mapping |= (i & 0xF) << (8 * (i & 0x3));
2430 		if ((i & 0x3) == 0x3) {
2431 			switch (hw->mac.type) {
2432 			case ixgbe_mac_82598EB:
2433 				IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2),
2434 				    ring_mapping);
2435 				break;
2436 
2437 			case ixgbe_mac_82599EB:
2438 			case ixgbe_mac_X540:
2439 				IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2),
2440 				    ring_mapping);
2441 				break;
2442 
2443 			default:
2444 				break;
2445 			}
2446 
2447 			ring_mapping = 0;
2448 		}
2449 	}
2450 	if (i & 0x3) {
2451 		switch (hw->mac.type) {
2452 		case ixgbe_mac_82598EB:
2453 			IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping);
2454 			break;
2455 
2456 		case ixgbe_mac_82599EB:
2457 		case ixgbe_mac_X540:
2458 			IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping);
2459 			break;
2460 
2461 		default:
2462 			break;
2463 		}
2464 	}
2465 
2466 	/*
2467 	 * Enable CRC appending and TX padding (for short tx frames)
2468 	 */
2469 	reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2470 	reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN;
2471 	IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2472 
2473 	/*
2474 	 * enable DMA for 82599 and X540 parts
2475 	 */
2476 	if (hw->mac.type == ixgbe_mac_82599EB ||
2477 	    hw->mac.type == ixgbe_mac_X540) {
2478 		/* DMATXCTL.TE must be set after all Tx config is complete */
2479 		reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2480 		reg_val |= IXGBE_DMATXCTL_TE;
2481 		IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val);
2482 
2483 		/* Disable arbiter to set MTQC */
2484 		reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2485 		reg_val |= IXGBE_RTTDCS_ARBDIS;
2486 		IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2487 		IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2488 		reg_val &= ~IXGBE_RTTDCS_ARBDIS;
2489 		IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2490 	}
2491 
2492 	/*
2493 	 * Enabling tx queues ..
2494 	 * For 82599 must be done after DMATXCTL.TE is set
2495 	 */
2496 	for (i = 0; i < ixgbe->num_tx_rings; i++) {
2497 		tx_ring = &ixgbe->tx_rings[i];
2498 		reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index));
2499 		reg_val |= IXGBE_TXDCTL_ENABLE;
2500 		IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val);
2501 	}
2502 }
2503 
2504 /*
2505  * ixgbe_setup_rss - Setup receive-side scaling feature.
2506  */
2507 static void
2508 ixgbe_setup_rss(ixgbe_t *ixgbe)
2509 {
2510 	struct ixgbe_hw *hw = &ixgbe->hw;
2511 	uint32_t i, mrqc, rxcsum;
2512 	uint32_t random;
2513 	uint32_t reta;
2514 	uint32_t ring_per_group;
2515 
2516 	/*
2517 	 * Fill out redirection table
2518 	 */
2519 	reta = 0;
2520 	ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2521 
2522 	for (i = 0; i < 128; i++) {
2523 		reta = (reta << 8) | (i % ring_per_group) |
2524 		    ((i % ring_per_group) << 4);
2525 		if ((i & 3) == 3)
2526 			IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2527 	}
2528 
2529 	/*
2530 	 * Fill out hash function seeds with a random constant
2531 	 */
2532 	for (i = 0; i < 10; i++) {
2533 		(void) random_get_pseudo_bytes((uint8_t *)&random,
2534 		    sizeof (uint32_t));
2535 		IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
2536 	}
2537 
2538 	/*
2539 	 * Enable RSS & perform hash on these packet types
2540 	 */
2541 	mrqc = IXGBE_MRQC_RSSEN |
2542 	    IXGBE_MRQC_RSS_FIELD_IPV4 |
2543 	    IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2544 	    IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2545 	    IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2546 	    IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2547 	    IXGBE_MRQC_RSS_FIELD_IPV6 |
2548 	    IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2549 	    IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2550 	    IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2551 	IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2552 
2553 	/*
2554 	 * Disable Packet Checksum to enable RSS for multiple receive queues.
2555 	 * It is an adapter hardware limitation that Packet Checksum is
2556 	 * mutually exclusive with RSS.
2557 	 */
2558 	rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2559 	rxcsum |= IXGBE_RXCSUM_PCSD;
2560 	rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
2561 	IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2562 }
2563 
2564 /*
2565  * ixgbe_setup_vmdq - Setup MAC classification feature
2566  */
2567 static void
2568 ixgbe_setup_vmdq(ixgbe_t *ixgbe)
2569 {
2570 	struct ixgbe_hw *hw = &ixgbe->hw;
2571 	uint32_t vmdctl, i, vtctl;
2572 
2573 	/*
2574 	 * Setup the VMDq Control register, enable VMDq based on
2575 	 * packet destination MAC address:
2576 	 */
2577 	switch (hw->mac.type) {
2578 	case ixgbe_mac_82598EB:
2579 		/*
2580 		 * VMDq Enable = 1;
2581 		 * VMDq Filter = 0; MAC filtering
2582 		 * Default VMDq output index = 0;
2583 		 */
2584 		vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2585 		IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2586 		break;
2587 
2588 	case ixgbe_mac_82599EB:
2589 	case ixgbe_mac_X540:
2590 		/*
2591 		 * Enable VMDq-only.
2592 		 */
2593 		vmdctl = IXGBE_MRQC_VMDQEN;
2594 		IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl);
2595 
2596 		for (i = 0; i < hw->mac.num_rar_entries; i++) {
2597 			IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2598 			IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2599 		}
2600 
2601 		/*
2602 		 * Enable Virtualization and Replication.
2603 		 */
2604 		vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2605 		IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2606 
2607 		/*
2608 		 * Enable receiving packets to all VFs
2609 		 */
2610 		IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2611 		IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2612 		break;
2613 
2614 	default:
2615 		break;
2616 	}
2617 }
2618 
2619 /*
2620  * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature.
2621  */
2622 static void
2623 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe)
2624 {
2625 	struct ixgbe_hw *hw = &ixgbe->hw;
2626 	uint32_t i, mrqc, rxcsum;
2627 	uint32_t random;
2628 	uint32_t reta;
2629 	uint32_t ring_per_group;
2630 	uint32_t vmdctl, vtctl;
2631 
2632 	/*
2633 	 * Fill out redirection table
2634 	 */
2635 	reta = 0;
2636 	ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2637 	for (i = 0; i < 128; i++) {
2638 		reta = (reta << 8) | (i % ring_per_group) |
2639 		    ((i % ring_per_group) << 4);
2640 		if ((i & 3) == 3)
2641 			IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2642 	}
2643 
2644 	/*
2645 	 * Fill out hash function seeds with a random constant
2646 	 */
2647 	for (i = 0; i < 10; i++) {
2648 		(void) random_get_pseudo_bytes((uint8_t *)&random,
2649 		    sizeof (uint32_t));
2650 		IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
2651 	}
2652 
2653 	/*
2654 	 * Enable and setup RSS and VMDq
2655 	 */
2656 	switch (hw->mac.type) {
2657 	case ixgbe_mac_82598EB:
2658 		/*
2659 		 * Enable RSS & Setup RSS Hash functions
2660 		 */
2661 		mrqc = IXGBE_MRQC_RSSEN |
2662 		    IXGBE_MRQC_RSS_FIELD_IPV4 |
2663 		    IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2664 		    IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2665 		    IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2666 		    IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2667 		    IXGBE_MRQC_RSS_FIELD_IPV6 |
2668 		    IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2669 		    IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2670 		    IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2671 		IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2672 
2673 		/*
2674 		 * Enable and Setup VMDq
2675 		 * VMDq Filter = 0; MAC filtering
2676 		 * Default VMDq output index = 0;
2677 		 */
2678 		vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2679 		IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2680 		break;
2681 
2682 	case ixgbe_mac_82599EB:
2683 	case ixgbe_mac_X540:
2684 		/*
2685 		 * Enable RSS & Setup RSS Hash functions
2686 		 */
2687 		mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 |
2688 		    IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2689 		    IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2690 		    IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2691 		    IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2692 		    IXGBE_MRQC_RSS_FIELD_IPV6 |
2693 		    IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2694 		    IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2695 		    IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2696 
2697 		/*
2698 		 * Enable VMDq+RSS.
2699 		 */
2700 		if (ixgbe->num_rx_groups > 32)  {
2701 			mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN;
2702 		} else {
2703 			mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN;
2704 		}
2705 
2706 		IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2707 
2708 		for (i = 0; i < hw->mac.num_rar_entries; i++) {
2709 			IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2710 			IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2711 		}
2712 		break;
2713 
2714 	default:
2715 		break;
2716 
2717 	}
2718 
2719 	/*
2720 	 * Disable Packet Checksum to enable RSS for multiple receive queues.
2721 	 * It is an adapter hardware limitation that Packet Checksum is
2722 	 * mutually exclusive with RSS.
2723 	 */
2724 	rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2725 	rxcsum |= IXGBE_RXCSUM_PCSD;
2726 	rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
2727 	IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2728 
2729 	if (hw->mac.type == ixgbe_mac_82599EB ||
2730 	    hw->mac.type == ixgbe_mac_X540) {
2731 		/*
2732 		 * Enable Virtualization and Replication.
2733 		 */
2734 		vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2735 		IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2736 
2737 		/*
2738 		 * Enable receiving packets to all VFs
2739 		 */
2740 		IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2741 		IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2742 	}
2743 }
2744 
2745 /*
2746  * ixgbe_init_unicst - Initialize the unicast addresses.
2747  */
2748 static void
2749 ixgbe_init_unicst(ixgbe_t *ixgbe)
2750 {
2751 	struct ixgbe_hw *hw = &ixgbe->hw;
2752 	uint8_t *mac_addr;
2753 	int slot;
2754 	/*
2755 	 * Here we should consider two situations:
2756 	 *
2757 	 * 1. Chipset is initialized at the first time,
2758 	 *    Clear all the multiple unicast addresses.
2759 	 *
2760 	 * 2. Chipset is reset
2761 	 *    Recover the multiple unicast addresses from the
2762 	 *    software data structure to the RAR registers.
2763 	 */
2764 	if (!ixgbe->unicst_init) {
2765 		/*
2766 		 * Initialize the multiple unicast addresses
2767 		 */
2768 		ixgbe->unicst_total = hw->mac.num_rar_entries;
2769 		ixgbe->unicst_avail = ixgbe->unicst_total;
2770 		for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2771 			mac_addr = ixgbe->unicst_addr[slot].mac.addr;
2772 			bzero(mac_addr, ETHERADDRL);
2773 			(void) ixgbe_set_rar(hw, slot, mac_addr, NULL, NULL);
2774 			ixgbe->unicst_addr[slot].mac.set = 0;
2775 		}
2776 		ixgbe->unicst_init = B_TRUE;
2777 	} else {
2778 		/* Re-configure the RAR registers */
2779 		for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2780 			mac_addr = ixgbe->unicst_addr[slot].mac.addr;
2781 			if (ixgbe->unicst_addr[slot].mac.set == 1) {
2782 				(void) ixgbe_set_rar(hw, slot, mac_addr,
2783 				    ixgbe->unicst_addr[slot].mac.group_index,
2784 				    IXGBE_RAH_AV);
2785 			} else {
2786 				bzero(mac_addr, ETHERADDRL);
2787 				(void) ixgbe_set_rar(hw, slot, mac_addr,
2788 				    NULL, NULL);
2789 			}
2790 		}
2791 	}
2792 }
2793 
2794 /*
2795  * ixgbe_unicst_find - Find the slot for the specified unicast address
2796  */
2797 int
2798 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr)
2799 {
2800 	int slot;
2801 
2802 	ASSERT(mutex_owned(&ixgbe->gen_lock));
2803 
2804 	for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2805 		if (bcmp(ixgbe->unicst_addr[slot].mac.addr,
2806 		    mac_addr, ETHERADDRL) == 0)
2807 			return (slot);
2808 	}
2809 
2810 	return (-1);
2811 }
2812 
2813 /*
2814  * ixgbe_multicst_add - Add a multicst address.
2815  */
2816 int
2817 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr)
2818 {
2819 	ASSERT(mutex_owned(&ixgbe->gen_lock));
2820 
2821 	if ((multiaddr[0] & 01) == 0) {
2822 		return (EINVAL);
2823 	}
2824 
2825 	if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) {
2826 		return (ENOENT);
2827 	}
2828 
2829 	bcopy(multiaddr,
2830 	    &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL);
2831 	ixgbe->mcast_count++;
2832 
2833 	/*
2834 	 * Update the multicast table in the hardware
2835 	 */
2836 	ixgbe_setup_multicst(ixgbe);
2837 
2838 	if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
2839 		ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
2840 		return (EIO);
2841 	}
2842 
2843 	return (0);
2844 }
2845 
2846 /*
2847  * ixgbe_multicst_remove - Remove a multicst address.
2848  */
2849 int
2850 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr)
2851 {
2852 	int i;
2853 
2854 	ASSERT(mutex_owned(&ixgbe->gen_lock));
2855 
2856 	for (i = 0; i < ixgbe->mcast_count; i++) {
2857 		if (bcmp(multiaddr, &ixgbe->mcast_table[i],
2858 		    ETHERADDRL) == 0) {
2859 			for (i++; i < ixgbe->mcast_count; i++) {
2860 				ixgbe->mcast_table[i - 1] =
2861 				    ixgbe->mcast_table[i];
2862 			}
2863 			ixgbe->mcast_count--;
2864 			break;
2865 		}
2866 	}
2867 
2868 	/*
2869 	 * Update the multicast table in the hardware
2870 	 */
2871 	ixgbe_setup_multicst(ixgbe);
2872 
2873 	if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
2874 		ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
2875 		return (EIO);
2876 	}
2877 
2878 	return (0);
2879 }
2880 
2881 /*
2882  * ixgbe_setup_multicast - Setup multicast data structures.
2883  *
2884  * This routine initializes all of the multicast related structures
2885  * and save them in the hardware registers.
2886  */
2887 static void
2888 ixgbe_setup_multicst(ixgbe_t *ixgbe)
2889 {
2890 	uint8_t *mc_addr_list;
2891 	uint32_t mc_addr_count;
2892 	struct ixgbe_hw *hw = &ixgbe->hw;
2893 
2894 	ASSERT(mutex_owned(&ixgbe->gen_lock));
2895 
2896 	ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES);
2897 
2898 	mc_addr_list = (uint8_t *)ixgbe->mcast_table;
2899 	mc_addr_count = ixgbe->mcast_count;
2900 
2901 	/*
2902 	 * Update the multicast addresses to the MTA registers
2903 	 */
2904 	(void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count,
2905 	    ixgbe_mc_table_itr, TRUE);
2906 }
2907 
2908 /*
2909  * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode).
2910  *
2911  * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers.
2912  * Different chipsets may have different allowed configuration of vmdq and rss.
2913  */
2914 static void
2915 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe)
2916 {
2917 	struct ixgbe_hw *hw = &ixgbe->hw;
2918 	uint32_t ring_per_group;
2919 
2920 	switch (hw->mac.type) {
2921 	case ixgbe_mac_82598EB:
2922 		/*
2923 		 * 82598 supports the following combination:
2924 		 * vmdq no. x rss no.
2925 		 * [5..16]  x 1
2926 		 * [1..4]   x [1..16]
2927 		 * However 8 rss queue per pool (vmdq) is sufficient for
2928 		 * most cases.
2929 		 */
2930 		ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2931 		if (ixgbe->num_rx_groups > 4) {
2932 			ixgbe->num_rx_rings = ixgbe->num_rx_groups;
2933 		} else {
2934 			ixgbe->num_rx_rings = ixgbe->num_rx_groups *
2935 			    min(8, ring_per_group);
2936 		}
2937 
2938 		break;
2939 
2940 	case ixgbe_mac_82599EB:
2941 	case ixgbe_mac_X540:
2942 		/*
2943 		 * 82599 supports the following combination:
2944 		 * vmdq no. x rss no.
2945 		 * [33..64] x [1..2]
2946 		 * [2..32]  x [1..4]
2947 		 * 1 x [1..16]
2948 		 * However 8 rss queue per pool (vmdq) is sufficient for
2949 		 * most cases.
2950 		 *
2951 		 * For now, treat X540 like the 82599.
2952 		 */
2953 		ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2954 		if (ixgbe->num_rx_groups == 1) {
2955 			ixgbe->num_rx_rings = min(8, ring_per_group);
2956 		} else if (ixgbe->num_rx_groups <= 32) {
2957 			ixgbe->num_rx_rings = ixgbe->num_rx_groups *
2958 			    min(4, ring_per_group);
2959 		} else if (ixgbe->num_rx_groups <= 64) {
2960 			ixgbe->num_rx_rings = ixgbe->num_rx_groups *
2961 			    min(2, ring_per_group);
2962 		}
2963 		break;
2964 
2965 	default:
2966 		break;
2967 	}
2968 
2969 	ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2970 
2971 	if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) {
2972 		ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
2973 	} else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) {
2974 		ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ;
2975 	} else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) {
2976 		ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS;
2977 	} else {
2978 		ixgbe->classify_mode = IXGBE_CLASSIFY_RSS;
2979 	}
2980 
2981 	IXGBE_DEBUGLOG_2(ixgbe, "rx group number:%d, rx ring number:%d",
2982 	    ixgbe->num_rx_groups, ixgbe->num_rx_rings);
2983 }
2984 
2985 /*
2986  * ixgbe_get_conf - Get driver configurations set in driver.conf.
2987  *
2988  * This routine gets user-configured values out of the configuration
2989  * file ixgbe.conf.
2990  *
2991  * For each configurable value, there is a minimum, a maximum, and a
2992  * default.
2993  * If user does not configure a value, use the default.
2994  * If user configures below the minimum, use the minumum.
2995  * If user configures above the maximum, use the maxumum.
2996  */
2997 static void
2998 ixgbe_get_conf(ixgbe_t *ixgbe)
2999 {
3000 	struct ixgbe_hw *hw = &ixgbe->hw;
3001 	uint32_t flow_control;
3002 
3003 	/*
3004 	 * ixgbe driver supports the following user configurations:
3005 	 *
3006 	 * Jumbo frame configuration:
3007 	 *    default_mtu
3008 	 *
3009 	 * Ethernet flow control configuration:
3010 	 *    flow_control
3011 	 *
3012 	 * Multiple rings configurations:
3013 	 *    tx_queue_number
3014 	 *    tx_ring_size
3015 	 *    rx_queue_number
3016 	 *    rx_ring_size
3017 	 *
3018 	 * Call ixgbe_get_prop() to get the value for a specific
3019 	 * configuration parameter.
3020 	 */
3021 
3022 	/*
3023 	 * Jumbo frame configuration - max_frame_size controls host buffer
3024 	 * allocation, so includes MTU, ethernet header, vlan tag and
3025 	 * frame check sequence.
3026 	 */
3027 	ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU,
3028 	    MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU);
3029 
3030 	ixgbe->max_frame_size = ixgbe->default_mtu +
3031 	    sizeof (struct ether_vlan_header) + ETHERFCSL;
3032 
3033 	/*
3034 	 * Ethernet flow control configuration
3035 	 */
3036 	flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL,
3037 	    ixgbe_fc_none, 3, ixgbe_fc_none);
3038 	if (flow_control == 3)
3039 		flow_control = ixgbe_fc_default;
3040 
3041 	/*
3042 	 * fc.requested mode is what the user requests.  After autoneg,
3043 	 * fc.current_mode will be the flow_control mode that was negotiated.
3044 	 */
3045 	hw->fc.requested_mode = flow_control;
3046 
3047 	/*
3048 	 * Multiple rings configurations
3049 	 */
3050 	ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM,
3051 	    ixgbe->capab->min_tx_que_num,
3052 	    ixgbe->capab->max_tx_que_num,
3053 	    ixgbe->capab->def_tx_que_num);
3054 	ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE,
3055 	    MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE);
3056 
3057 	ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM,
3058 	    ixgbe->capab->min_rx_que_num,
3059 	    ixgbe->capab->max_rx_que_num,
3060 	    ixgbe->capab->def_rx_que_num);
3061 	ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE,
3062 	    MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE);
3063 
3064 	/*
3065 	 * Multiple groups configuration
3066 	 */
3067 	ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM,
3068 	    ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num,
3069 	    ixgbe->capab->def_rx_grp_num);
3070 
3071 	ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE,
3072 	    0, 1, DEFAULT_MR_ENABLE);
3073 
3074 	if (ixgbe->mr_enable == B_FALSE) {
3075 		ixgbe->num_tx_rings = 1;
3076 		ixgbe->num_rx_rings = 1;
3077 		ixgbe->num_rx_groups = 1;
3078 		ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3079 	} else {
3080 		ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3081 		    max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1);
3082 		/*
3083 		 * The combination of num_rx_rings and num_rx_groups
3084 		 * may be not supported by h/w. We need to adjust
3085 		 * them to appropriate values.
3086 		 */
3087 		ixgbe_setup_vmdq_rss_conf(ixgbe);
3088 	}
3089 
3090 	/*
3091 	 * Tunable used to force an interrupt type. The only use is
3092 	 * for testing of the lesser interrupt types.
3093 	 * 0 = don't force interrupt type
3094 	 * 1 = force interrupt type MSI-X
3095 	 * 2 = force interrupt type MSI
3096 	 * 3 = force interrupt type Legacy
3097 	 */
3098 	ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE,
3099 	    IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE);
3100 
3101 	ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE,
3102 	    0, 1, DEFAULT_TX_HCKSUM_ENABLE);
3103 	ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE,
3104 	    0, 1, DEFAULT_RX_HCKSUM_ENABLE);
3105 	ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE,
3106 	    0, 1, DEFAULT_LSO_ENABLE);
3107 	ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE,
3108 	    0, 1, DEFAULT_LRO_ENABLE);
3109 	ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE,
3110 	    0, 1, DEFAULT_TX_HEAD_WB_ENABLE);
3111 	ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe,
3112 	    PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE);
3113 
3114 	/* Head Write Back not recommended for 82599 and X540 */
3115 	if (hw->mac.type == ixgbe_mac_82599EB ||
3116 	    hw->mac.type == ixgbe_mac_X540) {
3117 		ixgbe->tx_head_wb_enable = B_FALSE;
3118 	}
3119 
3120 	/*
3121 	 * ixgbe LSO needs the tx h/w checksum support.
3122 	 * LSO will be disabled if tx h/w checksum is not
3123 	 * enabled.
3124 	 */
3125 	if (ixgbe->tx_hcksum_enable == B_FALSE) {
3126 		ixgbe->lso_enable = B_FALSE;
3127 	}
3128 
3129 	/*
3130 	 * ixgbe LRO needs the rx h/w checksum support.
3131 	 * LRO will be disabled if rx h/w checksum is not
3132 	 * enabled.
3133 	 */
3134 	if (ixgbe->rx_hcksum_enable == B_FALSE) {
3135 		ixgbe->lro_enable = B_FALSE;
3136 	}
3137 
3138 	/*
3139 	 * ixgbe LRO only been supported by 82599 and X540 now
3140 	 */
3141 	if (hw->mac.type == ixgbe_mac_82598EB) {
3142 		ixgbe->lro_enable = B_FALSE;
3143 	}
3144 	ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD,
3145 	    MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
3146 	    DEFAULT_TX_COPY_THRESHOLD);
3147 	ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe,
3148 	    PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD,
3149 	    MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD);
3150 	ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe,
3151 	    PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD,
3152 	    MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD);
3153 	ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe,
3154 	    PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD,
3155 	    MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD);
3156 
3157 	ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD,
3158 	    MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
3159 	    DEFAULT_RX_COPY_THRESHOLD);
3160 	ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR,
3161 	    MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
3162 	    DEFAULT_RX_LIMIT_PER_INTR);
3163 
3164 	ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING,
3165 	    ixgbe->capab->min_intr_throttle,
3166 	    ixgbe->capab->max_intr_throttle,
3167 	    ixgbe->capab->def_intr_throttle);
3168 	/*
3169 	 * 82599 and X540 require the interrupt throttling rate is
3170 	 * a multiple of 8. This is enforced by the register
3171 	 * definiton.
3172 	 */
3173 	if (hw->mac.type == ixgbe_mac_82599EB || hw->mac.type == ixgbe_mac_X540)
3174 		ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8;
3175 }
3176 
3177 static void
3178 ixgbe_init_params(ixgbe_t *ixgbe)
3179 {
3180 	ixgbe->param_en_10000fdx_cap = 1;
3181 	ixgbe->param_en_1000fdx_cap = 1;
3182 	ixgbe->param_en_100fdx_cap = 1;
3183 	ixgbe->param_adv_10000fdx_cap = 1;
3184 	ixgbe->param_adv_1000fdx_cap = 1;
3185 	ixgbe->param_adv_100fdx_cap = 1;
3186 
3187 	ixgbe->param_pause_cap = 1;
3188 	ixgbe->param_asym_pause_cap = 1;
3189 	ixgbe->param_rem_fault = 0;
3190 
3191 	ixgbe->param_adv_autoneg_cap = 1;
3192 	ixgbe->param_adv_pause_cap = 1;
3193 	ixgbe->param_adv_asym_pause_cap = 1;
3194 	ixgbe->param_adv_rem_fault = 0;
3195 
3196 	ixgbe->param_lp_10000fdx_cap = 0;
3197 	ixgbe->param_lp_1000fdx_cap = 0;
3198 	ixgbe->param_lp_100fdx_cap = 0;
3199 	ixgbe->param_lp_autoneg_cap = 0;
3200 	ixgbe->param_lp_pause_cap = 0;
3201 	ixgbe->param_lp_asym_pause_cap = 0;
3202 	ixgbe->param_lp_rem_fault = 0;
3203 }
3204 
3205 /*
3206  * ixgbe_get_prop - Get a property value out of the configuration file
3207  * ixgbe.conf.
3208  *
3209  * Caller provides the name of the property, a default value, a minimum
3210  * value, and a maximum value.
3211  *
3212  * Return configured value of the property, with default, minimum and
3213  * maximum properly applied.
3214  */
3215 static int
3216 ixgbe_get_prop(ixgbe_t *ixgbe,
3217     char *propname,	/* name of the property */
3218     int minval,		/* minimum acceptable value */
3219     int maxval,		/* maximim acceptable value */
3220     int defval)		/* default value */
3221 {
3222 	int value;
3223 
3224 	/*
3225 	 * Call ddi_prop_get_int() to read the conf settings
3226 	 */
3227 	value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip,
3228 	    DDI_PROP_DONTPASS, propname, defval);
3229 	if (value > maxval)
3230 		value = maxval;
3231 
3232 	if (value < minval)
3233 		value = minval;
3234 
3235 	return (value);
3236 }
3237 
3238 /*
3239  * ixgbe_driver_setup_link - Using the link properties to setup the link.
3240  */
3241 int
3242 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw)
3243 {
3244 	u32 autoneg_advertised = 0;
3245 
3246 	/*
3247 	 * No half duplex support with 10Gb parts
3248 	 */
3249 	if (ixgbe->param_adv_10000fdx_cap == 1)
3250 		autoneg_advertised |= IXGBE_LINK_SPEED_10GB_FULL;
3251 
3252 	if (ixgbe->param_adv_1000fdx_cap == 1)
3253 		autoneg_advertised |= IXGBE_LINK_SPEED_1GB_FULL;
3254 
3255 	if (ixgbe->param_adv_100fdx_cap == 1)
3256 		autoneg_advertised |= IXGBE_LINK_SPEED_100_FULL;
3257 
3258 	if (ixgbe->param_adv_autoneg_cap == 1 && autoneg_advertised == 0) {
3259 		ixgbe_notice(ixgbe, "Invalid link settings. Setup link "
3260 		    "to autonegotiation with full link capabilities.");
3261 
3262 		autoneg_advertised = IXGBE_LINK_SPEED_10GB_FULL |
3263 		    IXGBE_LINK_SPEED_1GB_FULL |
3264 		    IXGBE_LINK_SPEED_100_FULL;
3265 	}
3266 
3267 	if (setup_hw) {
3268 		if (ixgbe_setup_link(&ixgbe->hw, autoneg_advertised,
3269 		    ixgbe->param_adv_autoneg_cap, B_TRUE) != IXGBE_SUCCESS) {
3270 			ixgbe_notice(ixgbe, "Setup link failed on this "
3271 			    "device.");
3272 			return (IXGBE_FAILURE);
3273 		}
3274 	}
3275 
3276 	return (IXGBE_SUCCESS);
3277 }
3278 
3279 /*
3280  * ixgbe_driver_link_check - Link status processing.
3281  *
3282  * This function can be called in both kernel context and interrupt context
3283  */
3284 static void
3285 ixgbe_driver_link_check(ixgbe_t *ixgbe)
3286 {
3287 	struct ixgbe_hw *hw = &ixgbe->hw;
3288 	ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
3289 	boolean_t link_up = B_FALSE;
3290 	boolean_t link_changed = B_FALSE;
3291 
3292 	ASSERT(mutex_owned(&ixgbe->gen_lock));
3293 
3294 	(void) ixgbe_check_link(hw, &speed, &link_up, false);
3295 	if (link_up) {
3296 		ixgbe->link_check_complete = B_TRUE;
3297 
3298 		/* Link is up, enable flow control settings */
3299 		(void) ixgbe_fc_enable(hw);
3300 
3301 		/*
3302 		 * The Link is up, check whether it was marked as down earlier
3303 		 */
3304 		if (ixgbe->link_state != LINK_STATE_UP) {
3305 			switch (speed) {
3306 			case IXGBE_LINK_SPEED_10GB_FULL:
3307 				ixgbe->link_speed = SPEED_10GB;
3308 				break;
3309 			case IXGBE_LINK_SPEED_1GB_FULL:
3310 				ixgbe->link_speed = SPEED_1GB;
3311 				break;
3312 			case IXGBE_LINK_SPEED_100_FULL:
3313 				ixgbe->link_speed = SPEED_100;
3314 			}
3315 			ixgbe->link_duplex = LINK_DUPLEX_FULL;
3316 			ixgbe->link_state = LINK_STATE_UP;
3317 			link_changed = B_TRUE;
3318 		}
3319 	} else {
3320 		if (ixgbe->link_check_complete == B_TRUE ||
3321 		    (ixgbe->link_check_complete == B_FALSE &&
3322 		    gethrtime() >= ixgbe->link_check_hrtime)) {
3323 			/*
3324 			 * The link is really down
3325 			 */
3326 			ixgbe->link_check_complete = B_TRUE;
3327 
3328 			if (ixgbe->link_state != LINK_STATE_DOWN) {
3329 				ixgbe->link_speed = 0;
3330 				ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN;
3331 				ixgbe->link_state = LINK_STATE_DOWN;
3332 				link_changed = B_TRUE;
3333 			}
3334 		}
3335 	}
3336 
3337 	/*
3338 	 * If we are in an interrupt context, need to re-enable the
3339 	 * interrupt, which was automasked
3340 	 */
3341 	if (servicing_interrupt() != 0) {
3342 		ixgbe->eims |= IXGBE_EICR_LSC;
3343 		IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3344 	}
3345 
3346 	if (link_changed) {
3347 		mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
3348 	}
3349 }
3350 
3351 /*
3352  * ixgbe_sfp_check - sfp module processing done in taskq only for 82599.
3353  */
3354 static void
3355 ixgbe_sfp_check(void *arg)
3356 {
3357 	ixgbe_t *ixgbe = (ixgbe_t *)arg;
3358 	uint32_t eicr = ixgbe->eicr;
3359 	struct ixgbe_hw *hw = &ixgbe->hw;
3360 
3361 	mutex_enter(&ixgbe->gen_lock);
3362 	if (eicr & IXGBE_EICR_GPI_SDP1) {
3363 		/* clear the interrupt */
3364 		IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
3365 
3366 		/* if link up, do multispeed fiber setup */
3367 		(void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3368 		    B_TRUE, B_TRUE);
3369 		ixgbe_driver_link_check(ixgbe);
3370 		ixgbe_get_hw_state(ixgbe);
3371 	} else if (eicr & IXGBE_EICR_GPI_SDP2) {
3372 		/* clear the interrupt */
3373 		IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
3374 
3375 		/* if link up, do sfp module setup */
3376 		(void) hw->mac.ops.setup_sfp(hw);
3377 
3378 		/* do multispeed fiber setup */
3379 		(void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3380 		    B_TRUE, B_TRUE);
3381 		ixgbe_driver_link_check(ixgbe);
3382 		ixgbe_get_hw_state(ixgbe);
3383 	}
3384 	mutex_exit(&ixgbe->gen_lock);
3385 
3386 	/*
3387 	 * We need to fully re-check the link later.
3388 	 */
3389 	ixgbe->link_check_complete = B_FALSE;
3390 	ixgbe->link_check_hrtime = gethrtime() +
3391 	    (IXGBE_LINK_UP_TIME * 100000000ULL);
3392 }
3393 
3394 /*
3395  * ixgbe_overtemp_check - overtemp module processing done in taskq
3396  *
3397  * This routine will only be called on adapters with temperature sensor.
3398  * The indication of over-temperature can be either SDP0 interrupt or the link
3399  * status change interrupt.
3400  */
3401 static void
3402 ixgbe_overtemp_check(void *arg)
3403 {
3404 	ixgbe_t *ixgbe = (ixgbe_t *)arg;
3405 	struct ixgbe_hw *hw = &ixgbe->hw;
3406 	uint32_t eicr = ixgbe->eicr;
3407 	ixgbe_link_speed speed;
3408 	boolean_t link_up;
3409 
3410 	mutex_enter(&ixgbe->gen_lock);
3411 
3412 	/* make sure we know current state of link */
3413 	(void) ixgbe_check_link(hw, &speed, &link_up, false);
3414 
3415 	/* check over-temp condition */
3416 	if (((eicr & IXGBE_EICR_GPI_SDP0) && (!link_up)) ||
3417 	    (eicr & IXGBE_EICR_LSC)) {
3418 		if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) {
3419 			atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3420 
3421 			/*
3422 			 * Disable the adapter interrupts
3423 			 */
3424 			ixgbe_disable_adapter_interrupts(ixgbe);
3425 
3426 			/*
3427 			 * Disable Rx/Tx units
3428 			 */
3429 			(void) ixgbe_stop_adapter(hw);
3430 
3431 			ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3432 			ixgbe_error(ixgbe,
3433 			    "Problem: Network adapter has been stopped "
3434 			    "because it has overheated");
3435 			ixgbe_error(ixgbe,
3436 			    "Action: Restart the computer. "
3437 			    "If the problem persists, power off the system "
3438 			    "and replace the adapter");
3439 		}
3440 	}
3441 
3442 	/* write to clear the interrupt */
3443 	IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
3444 
3445 	mutex_exit(&ixgbe->gen_lock);
3446 }
3447 
3448 /*
3449  * ixgbe_link_timer - timer for link status detection
3450  */
3451 static void
3452 ixgbe_link_timer(void *arg)
3453 {
3454 	ixgbe_t *ixgbe = (ixgbe_t *)arg;
3455 
3456 	mutex_enter(&ixgbe->gen_lock);
3457 	ixgbe_driver_link_check(ixgbe);
3458 	mutex_exit(&ixgbe->gen_lock);
3459 }
3460 
3461 /*
3462  * ixgbe_local_timer - Driver watchdog function.
3463  *
3464  * This function will handle the transmit stall check and other routines.
3465  */
3466 static void
3467 ixgbe_local_timer(void *arg)
3468 {
3469 	ixgbe_t *ixgbe = (ixgbe_t *)arg;
3470 
3471 	if (ixgbe->ixgbe_state & IXGBE_OVERTEMP)
3472 		goto out;
3473 
3474 	if (ixgbe->ixgbe_state & IXGBE_ERROR) {
3475 		ixgbe->reset_count++;
3476 		if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3477 			ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3478 		goto out;
3479 	}
3480 
3481 	if (ixgbe_stall_check(ixgbe)) {
3482 		atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL);
3483 		ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3484 
3485 		ixgbe->reset_count++;
3486 		if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3487 			ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3488 	}
3489 
3490 out:
3491 	ixgbe_restart_watchdog_timer(ixgbe);
3492 }
3493 
3494 /*
3495  * ixgbe_stall_check - Check for transmit stall.
3496  *
3497  * This function checks if the adapter is stalled (in transmit).
3498  *
3499  * It is called each time the watchdog timeout is invoked.
3500  * If the transmit descriptor reclaim continuously fails,
3501  * the watchdog value will increment by 1. If the watchdog
3502  * value exceeds the threshold, the ixgbe is assumed to
3503  * have stalled and need to be reset.
3504  */
3505 static boolean_t
3506 ixgbe_stall_check(ixgbe_t *ixgbe)
3507 {
3508 	ixgbe_tx_ring_t *tx_ring;
3509 	boolean_t result;
3510 	int i;
3511 
3512 	if (ixgbe->link_state != LINK_STATE_UP)
3513 		return (B_FALSE);
3514 
3515 	/*
3516 	 * If any tx ring is stalled, we'll reset the chipset
3517 	 */
3518 	result = B_FALSE;
3519 	for (i = 0; i < ixgbe->num_tx_rings; i++) {
3520 		tx_ring = &ixgbe->tx_rings[i];
3521 		if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) {
3522 			tx_ring->tx_recycle(tx_ring);
3523 		}
3524 
3525 		if (tx_ring->recycle_fail > 0)
3526 			tx_ring->stall_watchdog++;
3527 		else
3528 			tx_ring->stall_watchdog = 0;
3529 
3530 		if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) {
3531 			result = B_TRUE;
3532 			break;
3533 		}
3534 	}
3535 
3536 	if (result) {
3537 		tx_ring->stall_watchdog = 0;
3538 		tx_ring->recycle_fail = 0;
3539 	}
3540 
3541 	return (result);
3542 }
3543 
3544 
3545 /*
3546  * is_valid_mac_addr - Check if the mac address is valid.
3547  */
3548 static boolean_t
3549 is_valid_mac_addr(uint8_t *mac_addr)
3550 {
3551 	const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 };
3552 	const uint8_t addr_test2[6] =
3553 	    { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
3554 
3555 	if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) ||
3556 	    !(bcmp(addr_test2, mac_addr, ETHERADDRL)))
3557 		return (B_FALSE);
3558 
3559 	return (B_TRUE);
3560 }
3561 
3562 static boolean_t
3563 ixgbe_find_mac_address(ixgbe_t *ixgbe)
3564 {
3565 #ifdef __sparc
3566 	struct ixgbe_hw *hw = &ixgbe->hw;
3567 	uchar_t *bytes;
3568 	struct ether_addr sysaddr;
3569 	uint_t nelts;
3570 	int err;
3571 	boolean_t found = B_FALSE;
3572 
3573 	/*
3574 	 * The "vendor's factory-set address" may already have
3575 	 * been extracted from the chip, but if the property
3576 	 * "local-mac-address" is set we use that instead.
3577 	 *
3578 	 * We check whether it looks like an array of 6
3579 	 * bytes (which it should, if OBP set it).  If we can't
3580 	 * make sense of it this way, we'll ignore it.
3581 	 */
3582 	err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
3583 	    DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts);
3584 	if (err == DDI_PROP_SUCCESS) {
3585 		if (nelts == ETHERADDRL) {
3586 			while (nelts--)
3587 				hw->mac.addr[nelts] = bytes[nelts];
3588 			found = B_TRUE;
3589 		}
3590 		ddi_prop_free(bytes);
3591 	}
3592 
3593 	/*
3594 	 * Look up the OBP property "local-mac-address?". If the user has set
3595 	 * 'local-mac-address? = false', use "the system address" instead.
3596 	 */
3597 	if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0,
3598 	    "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
3599 		if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
3600 			if (localetheraddr(NULL, &sysaddr) != 0) {
3601 				bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
3602 				found = B_TRUE;
3603 			}
3604 		}
3605 		ddi_prop_free(bytes);
3606 	}
3607 
3608 	/*
3609 	 * Finally(!), if there's a valid "mac-address" property (created
3610 	 * if we netbooted from this interface), we must use this instead
3611 	 * of any of the above to ensure that the NFS/install server doesn't
3612 	 * get confused by the address changing as Solaris takes over!
3613 	 */
3614 	err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
3615 	    DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
3616 	if (err == DDI_PROP_SUCCESS) {
3617 		if (nelts == ETHERADDRL) {
3618 			while (nelts--)
3619 				hw->mac.addr[nelts] = bytes[nelts];
3620 			found = B_TRUE;
3621 		}
3622 		ddi_prop_free(bytes);
3623 	}
3624 
3625 	if (found) {
3626 		bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
3627 		return (B_TRUE);
3628 	}
3629 #else
3630 	_NOTE(ARGUNUSED(ixgbe));
3631 #endif
3632 
3633 	return (B_TRUE);
3634 }
3635 
3636 #pragma inline(ixgbe_arm_watchdog_timer)
3637 static void
3638 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe)
3639 {
3640 	/*
3641 	 * Fire a watchdog timer
3642 	 */
3643 	ixgbe->watchdog_tid =
3644 	    timeout(ixgbe_local_timer,
3645 	    (void *)ixgbe, 1 * drv_usectohz(1000000));
3646 
3647 }
3648 
3649 /*
3650  * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer.
3651  */
3652 void
3653 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe)
3654 {
3655 	mutex_enter(&ixgbe->watchdog_lock);
3656 
3657 	if (!ixgbe->watchdog_enable) {
3658 		ixgbe->watchdog_enable = B_TRUE;
3659 		ixgbe->watchdog_start = B_TRUE;
3660 		ixgbe_arm_watchdog_timer(ixgbe);
3661 	}
3662 
3663 	mutex_exit(&ixgbe->watchdog_lock);
3664 }
3665 
3666 /*
3667  * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer.
3668  */
3669 void
3670 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe)
3671 {
3672 	timeout_id_t tid;
3673 
3674 	mutex_enter(&ixgbe->watchdog_lock);
3675 
3676 	ixgbe->watchdog_enable = B_FALSE;
3677 	ixgbe->watchdog_start = B_FALSE;
3678 	tid = ixgbe->watchdog_tid;
3679 	ixgbe->watchdog_tid = 0;
3680 
3681 	mutex_exit(&ixgbe->watchdog_lock);
3682 
3683 	if (tid != 0)
3684 		(void) untimeout(tid);
3685 }
3686 
3687 /*
3688  * ixgbe_start_watchdog_timer - Start the driver watchdog timer.
3689  */
3690 void
3691 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe)
3692 {
3693 	mutex_enter(&ixgbe->watchdog_lock);
3694 
3695 	if (ixgbe->watchdog_enable) {
3696 		if (!ixgbe->watchdog_start) {
3697 			ixgbe->watchdog_start = B_TRUE;
3698 			ixgbe_arm_watchdog_timer(ixgbe);
3699 		}
3700 	}
3701 
3702 	mutex_exit(&ixgbe->watchdog_lock);
3703 }
3704 
3705 /*
3706  * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer.
3707  */
3708 static void
3709 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe)
3710 {
3711 	mutex_enter(&ixgbe->watchdog_lock);
3712 
3713 	if (ixgbe->watchdog_start)
3714 		ixgbe_arm_watchdog_timer(ixgbe);
3715 
3716 	mutex_exit(&ixgbe->watchdog_lock);
3717 }
3718 
3719 /*
3720  * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer.
3721  */
3722 void
3723 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe)
3724 {
3725 	timeout_id_t tid;
3726 
3727 	mutex_enter(&ixgbe->watchdog_lock);
3728 
3729 	ixgbe->watchdog_start = B_FALSE;
3730 	tid = ixgbe->watchdog_tid;
3731 	ixgbe->watchdog_tid = 0;
3732 
3733 	mutex_exit(&ixgbe->watchdog_lock);
3734 
3735 	if (tid != 0)
3736 		(void) untimeout(tid);
3737 }
3738 
3739 /*
3740  * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts.
3741  */
3742 static void
3743 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe)
3744 {
3745 	struct ixgbe_hw *hw = &ixgbe->hw;
3746 
3747 	/*
3748 	 * mask all interrupts off
3749 	 */
3750 	IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff);
3751 
3752 	/*
3753 	 * for MSI-X, also disable autoclear
3754 	 */
3755 	if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
3756 		IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0);
3757 	}
3758 
3759 	IXGBE_WRITE_FLUSH(hw);
3760 }
3761 
3762 /*
3763  * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts.
3764  */
3765 static void
3766 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe)
3767 {
3768 	struct ixgbe_hw *hw = &ixgbe->hw;
3769 	uint32_t eiac, eiam;
3770 	uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
3771 
3772 	/* interrupt types to enable */
3773 	ixgbe->eims = IXGBE_EIMS_ENABLE_MASK;	/* shared code default */
3774 	ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER;	/* minus tcp timer */
3775 	ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */
3776 
3777 	/* enable automask on "other" causes that this adapter can generate */
3778 	eiam = ixgbe->capab->other_intr;
3779 
3780 	/*
3781 	 * msi-x mode
3782 	 */
3783 	if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
3784 		/* enable autoclear but not on bits 29:20 */
3785 		eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR);
3786 
3787 		/* general purpose interrupt enable */
3788 		gpie |= (IXGBE_GPIE_MSIX_MODE
3789 		    | IXGBE_GPIE_PBA_SUPPORT
3790 		    | IXGBE_GPIE_OCD
3791 		    | IXGBE_GPIE_EIAME);
3792 	/*
3793 	 * non-msi-x mode
3794 	 */
3795 	} else {
3796 
3797 		/* disable autoclear, leave gpie at default */
3798 		eiac = 0;
3799 
3800 		/*
3801 		 * General purpose interrupt enable.
3802 		 * For 82599 or X540, extended interrupt automask enable
3803 		 * only in MSI or MSI-X mode
3804 		 */
3805 		if ((hw->mac.type == ixgbe_mac_82598EB) ||
3806 		    (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) {
3807 			gpie |= IXGBE_GPIE_EIAME;
3808 		}
3809 	}
3810 
3811 	/* Enable specific "other" interrupt types */
3812 	switch (hw->mac.type) {
3813 	case ixgbe_mac_82598EB:
3814 		gpie |= ixgbe->capab->other_gpie;
3815 		break;
3816 
3817 	case ixgbe_mac_82599EB:
3818 	case ixgbe_mac_X540:
3819 		gpie |= ixgbe->capab->other_gpie;
3820 
3821 		/* Enable RSC Delay 8us when LRO enabled  */
3822 		if (ixgbe->lro_enable) {
3823 			gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT);
3824 		}
3825 		break;
3826 
3827 	default:
3828 		break;
3829 	}
3830 
3831 	/* write to interrupt control registers */
3832 	IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3833 	IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac);
3834 	IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam);
3835 	IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
3836 	IXGBE_WRITE_FLUSH(hw);
3837 }
3838 
3839 /*
3840  * ixgbe_loopback_ioctl - Loopback support.
3841  */
3842 enum ioc_reply
3843 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp)
3844 {
3845 	lb_info_sz_t *lbsp;
3846 	lb_property_t *lbpp;
3847 	uint32_t *lbmp;
3848 	uint32_t size;
3849 	uint32_t value;
3850 
3851 	if (mp->b_cont == NULL)
3852 		return (IOC_INVAL);
3853 
3854 	switch (iocp->ioc_cmd) {
3855 	default:
3856 		return (IOC_INVAL);
3857 
3858 	case LB_GET_INFO_SIZE:
3859 		size = sizeof (lb_info_sz_t);
3860 		if (iocp->ioc_count != size)
3861 			return (IOC_INVAL);
3862 
3863 		value = sizeof (lb_normal);
3864 		value += sizeof (lb_mac);
3865 		value += sizeof (lb_external);
3866 
3867 		lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr;
3868 		*lbsp = value;
3869 		break;
3870 
3871 	case LB_GET_INFO:
3872 		value = sizeof (lb_normal);
3873 		value += sizeof (lb_mac);
3874 		value += sizeof (lb_external);
3875 
3876 		size = value;
3877 		if (iocp->ioc_count != size)
3878 			return (IOC_INVAL);
3879 
3880 		value = 0;
3881 		lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr;
3882 
3883 		lbpp[value++] = lb_normal;
3884 		lbpp[value++] = lb_mac;
3885 		lbpp[value++] = lb_external;
3886 		break;
3887 
3888 	case LB_GET_MODE:
3889 		size = sizeof (uint32_t);
3890 		if (iocp->ioc_count != size)
3891 			return (IOC_INVAL);
3892 
3893 		lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3894 		*lbmp = ixgbe->loopback_mode;
3895 		break;
3896 
3897 	case LB_SET_MODE:
3898 		size = 0;
3899 		if (iocp->ioc_count != sizeof (uint32_t))
3900 			return (IOC_INVAL);
3901 
3902 		lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3903 		if (!ixgbe_set_loopback_mode(ixgbe, *lbmp))
3904 			return (IOC_INVAL);
3905 		break;
3906 	}
3907 
3908 	iocp->ioc_count = size;
3909 	iocp->ioc_error = 0;
3910 
3911 	if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
3912 		ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3913 		return (IOC_INVAL);
3914 	}
3915 
3916 	return (IOC_REPLY);
3917 }
3918 
3919 /*
3920  * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode.
3921  */
3922 static boolean_t
3923 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode)
3924 {
3925 	if (mode == ixgbe->loopback_mode)
3926 		return (B_TRUE);
3927 
3928 	ixgbe->loopback_mode = mode;
3929 
3930 	if (mode == IXGBE_LB_NONE) {
3931 		/*
3932 		 * Reset the chip
3933 		 */
3934 		(void) ixgbe_reset(ixgbe);
3935 		return (B_TRUE);
3936 	}
3937 
3938 	mutex_enter(&ixgbe->gen_lock);
3939 
3940 	switch (mode) {
3941 	default:
3942 		mutex_exit(&ixgbe->gen_lock);
3943 		return (B_FALSE);
3944 
3945 	case IXGBE_LB_EXTERNAL:
3946 		break;
3947 
3948 	case IXGBE_LB_INTERNAL_MAC:
3949 		ixgbe_set_internal_mac_loopback(ixgbe);
3950 		break;
3951 	}
3952 
3953 	mutex_exit(&ixgbe->gen_lock);
3954 
3955 	return (B_TRUE);
3956 }
3957 
3958 /*
3959  * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode.
3960  */
3961 static void
3962 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe)
3963 {
3964 	struct ixgbe_hw *hw;
3965 	uint32_t reg;
3966 	uint8_t atlas;
3967 
3968 	hw = &ixgbe->hw;
3969 
3970 	/*
3971 	 * Setup MAC loopback
3972 	 */
3973 	reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0);
3974 	reg |= IXGBE_HLREG0_LPBK;
3975 	IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg);
3976 
3977 	reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
3978 	reg &= ~IXGBE_AUTOC_LMS_MASK;
3979 	IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
3980 
3981 	/*
3982 	 * Disable Atlas Tx lanes to keep packets in loopback and not on wire
3983 	 */
3984 	switch (hw->mac.type) {
3985 	case ixgbe_mac_82598EB:
3986 		(void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
3987 		    &atlas);
3988 		atlas |= IXGBE_ATLAS_PDN_TX_REG_EN;
3989 		(void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
3990 		    atlas);
3991 
3992 		(void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
3993 		    &atlas);
3994 		atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL;
3995 		(void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
3996 		    atlas);
3997 
3998 		(void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
3999 		    &atlas);
4000 		atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL;
4001 		(void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4002 		    atlas);
4003 
4004 		(void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4005 		    &atlas);
4006 		atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL;
4007 		(void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4008 		    atlas);
4009 		break;
4010 
4011 	case ixgbe_mac_82599EB:
4012 	case ixgbe_mac_X540:
4013 		reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4014 		reg |= (IXGBE_AUTOC_FLU |
4015 		    IXGBE_AUTOC_10G_KX4);
4016 		IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4017 
4018 		(void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL,
4019 		    B_FALSE, B_TRUE);
4020 		break;
4021 
4022 	default:
4023 		break;
4024 	}
4025 }
4026 
4027 #pragma inline(ixgbe_intr_rx_work)
4028 /*
4029  * ixgbe_intr_rx_work - RX processing of ISR.
4030  */
4031 static void
4032 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring)
4033 {
4034 	mblk_t *mp;
4035 
4036 	mutex_enter(&rx_ring->rx_lock);
4037 
4038 	mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4039 	mutex_exit(&rx_ring->rx_lock);
4040 
4041 	if (mp != NULL)
4042 		mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4043 		    rx_ring->ring_gen_num);
4044 }
4045 
4046 #pragma inline(ixgbe_intr_tx_work)
4047 /*
4048  * ixgbe_intr_tx_work - TX processing of ISR.
4049  */
4050 static void
4051 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring)
4052 {
4053 	ixgbe_t *ixgbe = tx_ring->ixgbe;
4054 
4055 	/*
4056 	 * Recycle the tx descriptors
4057 	 */
4058 	tx_ring->tx_recycle(tx_ring);
4059 
4060 	/*
4061 	 * Schedule the re-transmit
4062 	 */
4063 	if (tx_ring->reschedule &&
4064 	    (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) {
4065 		tx_ring->reschedule = B_FALSE;
4066 		mac_tx_ring_update(tx_ring->ixgbe->mac_hdl,
4067 		    tx_ring->ring_handle);
4068 		IXGBE_DEBUG_STAT(tx_ring->stat_reschedule);
4069 	}
4070 }
4071 
4072 #pragma inline(ixgbe_intr_other_work)
4073 /*
4074  * ixgbe_intr_other_work - Process interrupt types other than tx/rx
4075  */
4076 static void
4077 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr)
4078 {
4079 	ASSERT(mutex_owned(&ixgbe->gen_lock));
4080 
4081 	/*
4082 	 * handle link status change
4083 	 */
4084 	if (eicr & IXGBE_EICR_LSC) {
4085 		ixgbe_driver_link_check(ixgbe);
4086 		ixgbe_get_hw_state(ixgbe);
4087 	}
4088 
4089 	/*
4090 	 * check for fan failure on adapters with fans
4091 	 */
4092 	if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
4093 	    (eicr & IXGBE_EICR_GPI_SDP1)) {
4094 		atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
4095 
4096 		/*
4097 		 * Disable the adapter interrupts
4098 		 */
4099 		ixgbe_disable_adapter_interrupts(ixgbe);
4100 
4101 		/*
4102 		 * Disable Rx/Tx units
4103 		 */
4104 		(void) ixgbe_stop_adapter(&ixgbe->hw);
4105 
4106 		ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
4107 		ixgbe_error(ixgbe,
4108 		    "Problem: Network adapter has been stopped "
4109 		    "because the fan has stopped.\n");
4110 		ixgbe_error(ixgbe,
4111 		    "Action: Replace the adapter.\n");
4112 
4113 		/* re-enable the interrupt, which was automasked */
4114 		ixgbe->eims |= IXGBE_EICR_GPI_SDP1;
4115 	}
4116 
4117 	/*
4118 	 * Do SFP check for adapters with hot-plug capability
4119 	 */
4120 	if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) &&
4121 	    ((eicr & IXGBE_EICR_GPI_SDP1) || (eicr & IXGBE_EICR_GPI_SDP2))) {
4122 		ixgbe->eicr = eicr;
4123 		if ((ddi_taskq_dispatch(ixgbe->sfp_taskq,
4124 		    ixgbe_sfp_check, (void *)ixgbe,
4125 		    DDI_NOSLEEP)) != DDI_SUCCESS) {
4126 			ixgbe_log(ixgbe, "No memory available to dispatch "
4127 			    "taskq for SFP check");
4128 		}
4129 	}
4130 
4131 	/*
4132 	 * Do over-temperature check for adapters with temp sensor
4133 	 */
4134 	if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) &&
4135 	    ((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC))) {
4136 		ixgbe->eicr = eicr;
4137 		if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq,
4138 		    ixgbe_overtemp_check, (void *)ixgbe,
4139 		    DDI_NOSLEEP)) != DDI_SUCCESS) {
4140 			ixgbe_log(ixgbe, "No memory available to dispatch "
4141 			    "taskq for overtemp check");
4142 		}
4143 	}
4144 }
4145 
4146 /*
4147  * ixgbe_intr_legacy - Interrupt handler for legacy interrupts.
4148  */
4149 static uint_t
4150 ixgbe_intr_legacy(void *arg1, void *arg2)
4151 {
4152 	ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4153 	struct ixgbe_hw *hw = &ixgbe->hw;
4154 	ixgbe_tx_ring_t *tx_ring;
4155 	ixgbe_rx_ring_t *rx_ring;
4156 	uint32_t eicr;
4157 	mblk_t *mp;
4158 	boolean_t tx_reschedule;
4159 	uint_t result;
4160 
4161 	_NOTE(ARGUNUSED(arg2));
4162 
4163 	mutex_enter(&ixgbe->gen_lock);
4164 	if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
4165 		mutex_exit(&ixgbe->gen_lock);
4166 		return (DDI_INTR_UNCLAIMED);
4167 	}
4168 
4169 	mp = NULL;
4170 	tx_reschedule = B_FALSE;
4171 
4172 	/*
4173 	 * Any bit set in eicr: claim this interrupt
4174 	 */
4175 	eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4176 
4177 	if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4178 		mutex_exit(&ixgbe->gen_lock);
4179 		ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4180 		atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4181 		return (DDI_INTR_CLAIMED);
4182 	}
4183 
4184 	if (eicr) {
4185 		/*
4186 		 * For legacy interrupt, we have only one interrupt,
4187 		 * so we have only one rx ring and one tx ring enabled.
4188 		 */
4189 		ASSERT(ixgbe->num_rx_rings == 1);
4190 		ASSERT(ixgbe->num_tx_rings == 1);
4191 
4192 		/*
4193 		 * For legacy interrupt, rx rings[0] will use RTxQ[0].
4194 		 */
4195 		if (eicr & 0x1) {
4196 			ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE;
4197 			IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4198 			ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4199 			/*
4200 			 * Clean the rx descriptors
4201 			 */
4202 			rx_ring = &ixgbe->rx_rings[0];
4203 			mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4204 		}
4205 
4206 		/*
4207 		 * For legacy interrupt, tx rings[0] will use RTxQ[1].
4208 		 */
4209 		if (eicr & 0x2) {
4210 			/*
4211 			 * Recycle the tx descriptors
4212 			 */
4213 			tx_ring = &ixgbe->tx_rings[0];
4214 			tx_ring->tx_recycle(tx_ring);
4215 
4216 			/*
4217 			 * Schedule the re-transmit
4218 			 */
4219 			tx_reschedule = (tx_ring->reschedule &&
4220 			    (tx_ring->tbd_free >= ixgbe->tx_resched_thresh));
4221 		}
4222 
4223 		/* any interrupt type other than tx/rx */
4224 		if (eicr & ixgbe->capab->other_intr) {
4225 			switch (hw->mac.type) {
4226 			case ixgbe_mac_82598EB:
4227 				ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4228 				break;
4229 
4230 			case ixgbe_mac_82599EB:
4231 			case ixgbe_mac_X540:
4232 				ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4233 				IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4234 				break;
4235 
4236 			default:
4237 				break;
4238 			}
4239 			ixgbe_intr_other_work(ixgbe, eicr);
4240 			ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4241 		}
4242 
4243 		mutex_exit(&ixgbe->gen_lock);
4244 
4245 		result = DDI_INTR_CLAIMED;
4246 	} else {
4247 		mutex_exit(&ixgbe->gen_lock);
4248 
4249 		/*
4250 		 * No interrupt cause bits set: don't claim this interrupt.
4251 		 */
4252 		result = DDI_INTR_UNCLAIMED;
4253 	}
4254 
4255 	/* re-enable the interrupts which were automasked */
4256 	IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4257 
4258 	/*
4259 	 * Do the following work outside of the gen_lock
4260 	 */
4261 	if (mp != NULL) {
4262 		mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4263 		    rx_ring->ring_gen_num);
4264 	}
4265 
4266 	if (tx_reschedule)  {
4267 		tx_ring->reschedule = B_FALSE;
4268 		mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle);
4269 		IXGBE_DEBUG_STAT(tx_ring->stat_reschedule);
4270 	}
4271 
4272 	return (result);
4273 }
4274 
4275 /*
4276  * ixgbe_intr_msi - Interrupt handler for MSI.
4277  */
4278 static uint_t
4279 ixgbe_intr_msi(void *arg1, void *arg2)
4280 {
4281 	ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4282 	struct ixgbe_hw *hw = &ixgbe->hw;
4283 	uint32_t eicr;
4284 
4285 	_NOTE(ARGUNUSED(arg2));
4286 
4287 	eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4288 
4289 	if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4290 		ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4291 		atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4292 		return (DDI_INTR_CLAIMED);
4293 	}
4294 
4295 	/*
4296 	 * For MSI interrupt, we have only one vector,
4297 	 * so we have only one rx ring and one tx ring enabled.
4298 	 */
4299 	ASSERT(ixgbe->num_rx_rings == 1);
4300 	ASSERT(ixgbe->num_tx_rings == 1);
4301 
4302 	/*
4303 	 * For MSI interrupt, rx rings[0] will use RTxQ[0].
4304 	 */
4305 	if (eicr & 0x1) {
4306 		ixgbe_intr_rx_work(&ixgbe->rx_rings[0]);
4307 	}
4308 
4309 	/*
4310 	 * For MSI interrupt, tx rings[0] will use RTxQ[1].
4311 	 */
4312 	if (eicr & 0x2) {
4313 		ixgbe_intr_tx_work(&ixgbe->tx_rings[0]);
4314 	}
4315 
4316 	/* any interrupt type other than tx/rx */
4317 	if (eicr & ixgbe->capab->other_intr) {
4318 		mutex_enter(&ixgbe->gen_lock);
4319 		switch (hw->mac.type) {
4320 		case ixgbe_mac_82598EB:
4321 			ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4322 			break;
4323 
4324 		case ixgbe_mac_82599EB:
4325 		case ixgbe_mac_X540:
4326 			ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4327 			IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4328 			break;
4329 
4330 		default:
4331 			break;
4332 		}
4333 		ixgbe_intr_other_work(ixgbe, eicr);
4334 		ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4335 		mutex_exit(&ixgbe->gen_lock);
4336 	}
4337 
4338 	/* re-enable the interrupts which were automasked */
4339 	IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4340 
4341 	return (DDI_INTR_CLAIMED);
4342 }
4343 
4344 /*
4345  * ixgbe_intr_msix - Interrupt handler for MSI-X.
4346  */
4347 static uint_t
4348 ixgbe_intr_msix(void *arg1, void *arg2)
4349 {
4350 	ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1;
4351 	ixgbe_t *ixgbe = vect->ixgbe;
4352 	struct ixgbe_hw *hw = &ixgbe->hw;
4353 	uint32_t eicr;
4354 	int r_idx = 0;
4355 
4356 	_NOTE(ARGUNUSED(arg2));
4357 
4358 	/*
4359 	 * Clean each rx ring that has its bit set in the map
4360 	 */
4361 	r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1));
4362 	while (r_idx >= 0) {
4363 		ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]);
4364 		r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
4365 		    (ixgbe->num_rx_rings - 1));
4366 	}
4367 
4368 	/*
4369 	 * Clean each tx ring that has its bit set in the map
4370 	 */
4371 	r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1));
4372 	while (r_idx >= 0) {
4373 		ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]);
4374 		r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
4375 		    (ixgbe->num_tx_rings - 1));
4376 	}
4377 
4378 
4379 	/*
4380 	 * Clean other interrupt (link change) that has its bit set in the map
4381 	 */
4382 	if (BT_TEST(vect->other_map, 0) == 1) {
4383 		eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4384 
4385 		if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) !=
4386 		    DDI_FM_OK) {
4387 			ddi_fm_service_impact(ixgbe->dip,
4388 			    DDI_SERVICE_DEGRADED);
4389 			atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4390 			return (DDI_INTR_CLAIMED);
4391 		}
4392 
4393 		/*
4394 		 * Check "other" cause bits: any interrupt type other than tx/rx
4395 		 */
4396 		if (eicr & ixgbe->capab->other_intr) {
4397 			mutex_enter(&ixgbe->gen_lock);
4398 			switch (hw->mac.type) {
4399 			case ixgbe_mac_82598EB:
4400 				ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4401 				ixgbe_intr_other_work(ixgbe, eicr);
4402 				break;
4403 
4404 			case ixgbe_mac_82599EB:
4405 			case ixgbe_mac_X540:
4406 				ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4407 				ixgbe_intr_other_work(ixgbe, eicr);
4408 				break;
4409 
4410 			default:
4411 				break;
4412 			}
4413 			mutex_exit(&ixgbe->gen_lock);
4414 		}
4415 
4416 		/* re-enable the interrupts which were automasked */
4417 		IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4418 	}
4419 
4420 	return (DDI_INTR_CLAIMED);
4421 }
4422 
4423 /*
4424  * ixgbe_alloc_intrs - Allocate interrupts for the driver.
4425  *
4426  * Normal sequence is to try MSI-X; if not sucessful, try MSI;
4427  * if not successful, try Legacy.
4428  * ixgbe->intr_force can be used to force sequence to start with
4429  * any of the 3 types.
4430  * If MSI-X is not used, number of tx/rx rings is forced to 1.
4431  */
4432 static int
4433 ixgbe_alloc_intrs(ixgbe_t *ixgbe)
4434 {
4435 	dev_info_t *devinfo;
4436 	int intr_types;
4437 	int rc;
4438 
4439 	devinfo = ixgbe->dip;
4440 
4441 	/*
4442 	 * Get supported interrupt types
4443 	 */
4444 	rc = ddi_intr_get_supported_types(devinfo, &intr_types);
4445 
4446 	if (rc != DDI_SUCCESS) {
4447 		ixgbe_log(ixgbe,
4448 		    "Get supported interrupt types failed: %d", rc);
4449 		return (IXGBE_FAILURE);
4450 	}
4451 	IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types);
4452 
4453 	ixgbe->intr_type = 0;
4454 
4455 	/*
4456 	 * Install MSI-X interrupts
4457 	 */
4458 	if ((intr_types & DDI_INTR_TYPE_MSIX) &&
4459 	    (ixgbe->intr_force <= IXGBE_INTR_MSIX)) {
4460 		rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX);
4461 		if (rc == IXGBE_SUCCESS)
4462 			return (IXGBE_SUCCESS);
4463 
4464 		ixgbe_log(ixgbe,
4465 		    "Allocate MSI-X failed, trying MSI interrupts...");
4466 	}
4467 
4468 	/*
4469 	 * MSI-X not used, force rings and groups to 1
4470 	 */
4471 	ixgbe->num_rx_rings = 1;
4472 	ixgbe->num_rx_groups = 1;
4473 	ixgbe->num_tx_rings = 1;
4474 	ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
4475 	ixgbe_log(ixgbe,
4476 	    "MSI-X not used, force rings and groups number to 1");
4477 
4478 	/*
4479 	 * Install MSI interrupts
4480 	 */
4481 	if ((intr_types & DDI_INTR_TYPE_MSI) &&
4482 	    (ixgbe->intr_force <= IXGBE_INTR_MSI)) {
4483 		rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI);
4484 		if (rc == IXGBE_SUCCESS)
4485 			return (IXGBE_SUCCESS);
4486 
4487 		ixgbe_log(ixgbe,
4488 		    "Allocate MSI failed, trying Legacy interrupts...");
4489 	}
4490 
4491 	/*
4492 	 * Install legacy interrupts
4493 	 */
4494 	if (intr_types & DDI_INTR_TYPE_FIXED) {
4495 		rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED);
4496 		if (rc == IXGBE_SUCCESS)
4497 			return (IXGBE_SUCCESS);
4498 
4499 		ixgbe_log(ixgbe,
4500 		    "Allocate Legacy interrupts failed");
4501 	}
4502 
4503 	/*
4504 	 * If none of the 3 types succeeded, return failure
4505 	 */
4506 	return (IXGBE_FAILURE);
4507 }
4508 
4509 /*
4510  * ixgbe_alloc_intr_handles - Allocate interrupt handles.
4511  *
4512  * For legacy and MSI, only 1 handle is needed.  For MSI-X,
4513  * if fewer than 2 handles are available, return failure.
4514  * Upon success, this maps the vectors to rx and tx rings for
4515  * interrupts.
4516  */
4517 static int
4518 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type)
4519 {
4520 	dev_info_t *devinfo;
4521 	int request, count, actual;
4522 	int minimum;
4523 	int rc;
4524 	uint32_t ring_per_group;
4525 
4526 	devinfo = ixgbe->dip;
4527 
4528 	switch (intr_type) {
4529 	case DDI_INTR_TYPE_FIXED:
4530 		request = 1;	/* Request 1 legacy interrupt handle */
4531 		minimum = 1;
4532 		IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy");
4533 		break;
4534 
4535 	case DDI_INTR_TYPE_MSI:
4536 		request = 1;	/* Request 1 MSI interrupt handle */
4537 		minimum = 1;
4538 		IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI");
4539 		break;
4540 
4541 	case DDI_INTR_TYPE_MSIX:
4542 		/*
4543 		 * Best number of vectors for the adapter is
4544 		 * (# rx rings + # tx rings), however we will
4545 		 * limit the request number.
4546 		 */
4547 		request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings);
4548 		if (request > ixgbe->capab->max_ring_vect)
4549 			request = ixgbe->capab->max_ring_vect;
4550 		minimum = 1;
4551 		IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X");
4552 		break;
4553 
4554 	default:
4555 		ixgbe_log(ixgbe,
4556 		    "invalid call to ixgbe_alloc_intr_handles(): %d\n",
4557 		    intr_type);
4558 		return (IXGBE_FAILURE);
4559 	}
4560 	IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d  minimum: %d",
4561 	    request, minimum);
4562 
4563 	/*
4564 	 * Get number of supported interrupts
4565 	 */
4566 	rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
4567 	if ((rc != DDI_SUCCESS) || (count < minimum)) {
4568 		ixgbe_log(ixgbe,
4569 		    "Get interrupt number failed. Return: %d, count: %d",
4570 		    rc, count);
4571 		return (IXGBE_FAILURE);
4572 	}
4573 	IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count);
4574 
4575 	actual = 0;
4576 	ixgbe->intr_cnt = 0;
4577 	ixgbe->intr_cnt_max = 0;
4578 	ixgbe->intr_cnt_min = 0;
4579 
4580 	/*
4581 	 * Allocate an array of interrupt handles
4582 	 */
4583 	ixgbe->intr_size = request * sizeof (ddi_intr_handle_t);
4584 	ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP);
4585 
4586 	rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0,
4587 	    request, &actual, DDI_INTR_ALLOC_NORMAL);
4588 	if (rc != DDI_SUCCESS) {
4589 		ixgbe_log(ixgbe, "Allocate interrupts failed. "
4590 		    "return: %d, request: %d, actual: %d",
4591 		    rc, request, actual);
4592 		goto alloc_handle_fail;
4593 	}
4594 	IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual);
4595 
4596 	/*
4597 	 * upper/lower limit of interrupts
4598 	 */
4599 	ixgbe->intr_cnt = actual;
4600 	ixgbe->intr_cnt_max = request;
4601 	ixgbe->intr_cnt_min = minimum;
4602 
4603 	/*
4604 	 * rss number per group should not exceed the rx interrupt number,
4605 	 * else need to adjust rx ring number.
4606 	 */
4607 	ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
4608 	ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0);
4609 	if (actual < ring_per_group) {
4610 		ixgbe->num_rx_rings = ixgbe->num_rx_groups * actual;
4611 		ixgbe_setup_vmdq_rss_conf(ixgbe);
4612 	}
4613 
4614 	/*
4615 	 * Now we know the actual number of vectors.  Here we map the vector
4616 	 * to other, rx rings and tx ring.
4617 	 */
4618 	if (actual < minimum) {
4619 		ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d",
4620 		    actual);
4621 		goto alloc_handle_fail;
4622 	}
4623 
4624 	/*
4625 	 * Get priority for first vector, assume remaining are all the same
4626 	 */
4627 	rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
4628 	if (rc != DDI_SUCCESS) {
4629 		ixgbe_log(ixgbe,
4630 		    "Get interrupt priority failed: %d", rc);
4631 		goto alloc_handle_fail;
4632 	}
4633 
4634 	rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
4635 	if (rc != DDI_SUCCESS) {
4636 		ixgbe_log(ixgbe,
4637 		    "Get interrupt cap failed: %d", rc);
4638 		goto alloc_handle_fail;
4639 	}
4640 
4641 	ixgbe->intr_type = intr_type;
4642 
4643 	return (IXGBE_SUCCESS);
4644 
4645 alloc_handle_fail:
4646 	ixgbe_rem_intrs(ixgbe);
4647 
4648 	return (IXGBE_FAILURE);
4649 }
4650 
4651 /*
4652  * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type.
4653  *
4654  * Before adding the interrupt handlers, the interrupt vectors have
4655  * been allocated, and the rx/tx rings have also been allocated.
4656  */
4657 static int
4658 ixgbe_add_intr_handlers(ixgbe_t *ixgbe)
4659 {
4660 	int vector = 0;
4661 	int rc;
4662 
4663 	switch (ixgbe->intr_type) {
4664 	case DDI_INTR_TYPE_MSIX:
4665 		/*
4666 		 * Add interrupt handler for all vectors
4667 		 */
4668 		for (vector = 0; vector < ixgbe->intr_cnt; vector++) {
4669 			/*
4670 			 * install pointer to vect_map[vector]
4671 			 */
4672 			rc = ddi_intr_add_handler(ixgbe->htable[vector],
4673 			    (ddi_intr_handler_t *)ixgbe_intr_msix,
4674 			    (void *)&ixgbe->vect_map[vector], NULL);
4675 
4676 			if (rc != DDI_SUCCESS) {
4677 				ixgbe_log(ixgbe,
4678 				    "Add interrupt handler failed. "
4679 				    "return: %d, vector: %d", rc, vector);
4680 				for (vector--; vector >= 0; vector--) {
4681 					(void) ddi_intr_remove_handler(
4682 					    ixgbe->htable[vector]);
4683 				}
4684 				return (IXGBE_FAILURE);
4685 			}
4686 		}
4687 
4688 		break;
4689 
4690 	case DDI_INTR_TYPE_MSI:
4691 		/*
4692 		 * Add interrupt handlers for the only vector
4693 		 */
4694 		rc = ddi_intr_add_handler(ixgbe->htable[vector],
4695 		    (ddi_intr_handler_t *)ixgbe_intr_msi,
4696 		    (void *)ixgbe, NULL);
4697 
4698 		if (rc != DDI_SUCCESS) {
4699 			ixgbe_log(ixgbe,
4700 			    "Add MSI interrupt handler failed: %d", rc);
4701 			return (IXGBE_FAILURE);
4702 		}
4703 
4704 		break;
4705 
4706 	case DDI_INTR_TYPE_FIXED:
4707 		/*
4708 		 * Add interrupt handlers for the only vector
4709 		 */
4710 		rc = ddi_intr_add_handler(ixgbe->htable[vector],
4711 		    (ddi_intr_handler_t *)ixgbe_intr_legacy,
4712 		    (void *)ixgbe, NULL);
4713 
4714 		if (rc != DDI_SUCCESS) {
4715 			ixgbe_log(ixgbe,
4716 			    "Add legacy interrupt handler failed: %d", rc);
4717 			return (IXGBE_FAILURE);
4718 		}
4719 
4720 		break;
4721 
4722 	default:
4723 		return (IXGBE_FAILURE);
4724 	}
4725 
4726 	return (IXGBE_SUCCESS);
4727 }
4728 
4729 #pragma inline(ixgbe_map_rxring_to_vector)
4730 /*
4731  * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector.
4732  */
4733 static void
4734 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx)
4735 {
4736 	/*
4737 	 * Set bit in map
4738 	 */
4739 	BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
4740 
4741 	/*
4742 	 * Count bits set
4743 	 */
4744 	ixgbe->vect_map[v_idx].rxr_cnt++;
4745 
4746 	/*
4747 	 * Remember bit position
4748 	 */
4749 	ixgbe->rx_rings[r_idx].intr_vector = v_idx;
4750 	ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx;
4751 }
4752 
4753 #pragma inline(ixgbe_map_txring_to_vector)
4754 /*
4755  * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector.
4756  */
4757 static void
4758 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx)
4759 {
4760 	/*
4761 	 * Set bit in map
4762 	 */
4763 	BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx);
4764 
4765 	/*
4766 	 * Count bits set
4767 	 */
4768 	ixgbe->vect_map[v_idx].txr_cnt++;
4769 
4770 	/*
4771 	 * Remember bit position
4772 	 */
4773 	ixgbe->tx_rings[t_idx].intr_vector = v_idx;
4774 	ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx;
4775 }
4776 
4777 /*
4778  * ixgbe_setup_ivar - Set the given entry in the given interrupt vector
4779  * allocation register (IVAR).
4780  * cause:
4781  *   -1 : other cause
4782  *    0 : rx
4783  *    1 : tx
4784  */
4785 static void
4786 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector,
4787     int8_t cause)
4788 {
4789 	struct ixgbe_hw *hw = &ixgbe->hw;
4790 	u32 ivar, index;
4791 
4792 	switch (hw->mac.type) {
4793 	case ixgbe_mac_82598EB:
4794 		msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4795 		if (cause == -1) {
4796 			cause = 0;
4797 		}
4798 		index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4799 		ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4800 		ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3)));
4801 		ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3)));
4802 		IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4803 		break;
4804 
4805 	case ixgbe_mac_82599EB:
4806 	case ixgbe_mac_X540:
4807 		if (cause == -1) {
4808 			/* other causes */
4809 			msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4810 			index = (intr_alloc_entry & 1) * 8;
4811 			ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4812 			ivar &= ~(0xFF << index);
4813 			ivar |= (msix_vector << index);
4814 			IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4815 		} else {
4816 			/* tx or rx causes */
4817 			msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4818 			index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4819 			ivar = IXGBE_READ_REG(hw,
4820 			    IXGBE_IVAR(intr_alloc_entry >> 1));
4821 			ivar &= ~(0xFF << index);
4822 			ivar |= (msix_vector << index);
4823 			IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4824 			    ivar);
4825 		}
4826 		break;
4827 
4828 	default:
4829 		break;
4830 	}
4831 }
4832 
4833 /*
4834  * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of
4835  * given interrupt vector allocation register (IVAR).
4836  * cause:
4837  *   -1 : other cause
4838  *    0 : rx
4839  *    1 : tx
4840  */
4841 static void
4842 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
4843 {
4844 	struct ixgbe_hw *hw = &ixgbe->hw;
4845 	u32 ivar, index;
4846 
4847 	switch (hw->mac.type) {
4848 	case ixgbe_mac_82598EB:
4849 		if (cause == -1) {
4850 			cause = 0;
4851 		}
4852 		index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4853 		ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4854 		ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 *
4855 		    (intr_alloc_entry & 0x3)));
4856 		IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4857 		break;
4858 
4859 	case ixgbe_mac_82599EB:
4860 	case ixgbe_mac_X540:
4861 		if (cause == -1) {
4862 			/* other causes */
4863 			index = (intr_alloc_entry & 1) * 8;
4864 			ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4865 			ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
4866 			IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4867 		} else {
4868 			/* tx or rx causes */
4869 			index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4870 			ivar = IXGBE_READ_REG(hw,
4871 			    IXGBE_IVAR(intr_alloc_entry >> 1));
4872 			ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
4873 			IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4874 			    ivar);
4875 		}
4876 		break;
4877 
4878 	default:
4879 		break;
4880 	}
4881 }
4882 
4883 /*
4884  * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of
4885  * given interrupt vector allocation register (IVAR).
4886  * cause:
4887  *   -1 : other cause
4888  *    0 : rx
4889  *    1 : tx
4890  */
4891 static void
4892 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
4893 {
4894 	struct ixgbe_hw *hw = &ixgbe->hw;
4895 	u32 ivar, index;
4896 
4897 	switch (hw->mac.type) {
4898 	case ixgbe_mac_82598EB:
4899 		if (cause == -1) {
4900 			cause = 0;
4901 		}
4902 		index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4903 		ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4904 		ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 *
4905 		    (intr_alloc_entry & 0x3)));
4906 		IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4907 		break;
4908 
4909 	case ixgbe_mac_82599EB:
4910 	case ixgbe_mac_X540:
4911 		if (cause == -1) {
4912 			/* other causes */
4913 			index = (intr_alloc_entry & 1) * 8;
4914 			ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4915 			ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
4916 			IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4917 		} else {
4918 			/* tx or rx causes */
4919 			index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4920 			ivar = IXGBE_READ_REG(hw,
4921 			    IXGBE_IVAR(intr_alloc_entry >> 1));
4922 			ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
4923 			IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4924 			    ivar);
4925 		}
4926 		break;
4927 
4928 	default:
4929 		break;
4930 	}
4931 }
4932 
4933 /*
4934  * Convert the rx ring index driver maintained to the rx ring index
4935  * in h/w.
4936  */
4937 static uint32_t
4938 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index)
4939 {
4940 
4941 	struct ixgbe_hw *hw = &ixgbe->hw;
4942 	uint32_t rx_ring_per_group, hw_rx_index;
4943 
4944 	if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS ||
4945 	    ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) {
4946 		return (sw_rx_index);
4947 	} else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) {
4948 		switch (hw->mac.type) {
4949 		case ixgbe_mac_82598EB:
4950 			return (sw_rx_index);
4951 
4952 		case ixgbe_mac_82599EB:
4953 		case ixgbe_mac_X540:
4954 			return (sw_rx_index * 2);
4955 
4956 		default:
4957 			break;
4958 		}
4959 	} else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) {
4960 		rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
4961 
4962 		switch (hw->mac.type) {
4963 		case ixgbe_mac_82598EB:
4964 			hw_rx_index = (sw_rx_index / rx_ring_per_group) *
4965 			    16 + (sw_rx_index % rx_ring_per_group);
4966 			return (hw_rx_index);
4967 
4968 		case ixgbe_mac_82599EB:
4969 		case ixgbe_mac_X540:
4970 			if (ixgbe->num_rx_groups > 32) {
4971 				hw_rx_index = (sw_rx_index /
4972 				    rx_ring_per_group) * 2 +
4973 				    (sw_rx_index % rx_ring_per_group);
4974 			} else {
4975 				hw_rx_index = (sw_rx_index /
4976 				    rx_ring_per_group) * 4 +
4977 				    (sw_rx_index % rx_ring_per_group);
4978 			}
4979 			return (hw_rx_index);
4980 
4981 		default:
4982 			break;
4983 		}
4984 	}
4985 
4986 	/*
4987 	 * Should never reach. Just to make compiler happy.
4988 	 */
4989 	return (sw_rx_index);
4990 }
4991 
4992 /*
4993  * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors.
4994  *
4995  * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt
4996  * to vector[0 - (intr_cnt -1)].
4997  */
4998 static int
4999 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe)
5000 {
5001 	int i, vector = 0;
5002 
5003 	/* initialize vector map */
5004 	bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
5005 	for (i = 0; i < ixgbe->intr_cnt; i++) {
5006 		ixgbe->vect_map[i].ixgbe = ixgbe;
5007 	}
5008 
5009 	/*
5010 	 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0],
5011 	 * tx rings[0] on RTxQ[1].
5012 	 */
5013 	if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5014 		ixgbe_map_rxring_to_vector(ixgbe, 0, 0);
5015 		ixgbe_map_txring_to_vector(ixgbe, 0, 1);
5016 		return (IXGBE_SUCCESS);
5017 	}
5018 
5019 	/*
5020 	 * Interrupts/vectors mapping for MSI-X
5021 	 */
5022 
5023 	/*
5024 	 * Map other interrupt to vector 0,
5025 	 * Set bit in map and count the bits set.
5026 	 */
5027 	BT_SET(ixgbe->vect_map[vector].other_map, 0);
5028 	ixgbe->vect_map[vector].other_cnt++;
5029 
5030 	/*
5031 	 * Map rx ring interrupts to vectors
5032 	 */
5033 	for (i = 0; i < ixgbe->num_rx_rings; i++) {
5034 		ixgbe_map_rxring_to_vector(ixgbe, i, vector);
5035 		vector = (vector +1) % ixgbe->intr_cnt;
5036 	}
5037 
5038 	/*
5039 	 * Map tx ring interrupts to vectors
5040 	 */
5041 	for (i = 0; i < ixgbe->num_tx_rings; i++) {
5042 		ixgbe_map_txring_to_vector(ixgbe, i, vector);
5043 		vector = (vector +1) % ixgbe->intr_cnt;
5044 	}
5045 
5046 	return (IXGBE_SUCCESS);
5047 }
5048 
5049 /*
5050  * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s).
5051  *
5052  * This relies on ring/vector mapping already set up in the
5053  * vect_map[] structures
5054  */
5055 static void
5056 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe)
5057 {
5058 	struct ixgbe_hw *hw = &ixgbe->hw;
5059 	ixgbe_intr_vector_t *vect;	/* vector bitmap */
5060 	int r_idx;	/* ring index */
5061 	int v_idx;	/* vector index */
5062 	uint32_t hw_index;
5063 
5064 	/*
5065 	 * Clear any previous entries
5066 	 */
5067 	switch (hw->mac.type) {
5068 	case ixgbe_mac_82598EB:
5069 		for (v_idx = 0; v_idx < 25; v_idx++)
5070 			IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5071 		break;
5072 
5073 	case ixgbe_mac_82599EB:
5074 	case ixgbe_mac_X540:
5075 		for (v_idx = 0; v_idx < 64; v_idx++)
5076 			IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5077 		IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0);
5078 		break;
5079 
5080 	default:
5081 		break;
5082 	}
5083 
5084 	/*
5085 	 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and
5086 	 * tx rings[0] will use RTxQ[1].
5087 	 */
5088 	if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5089 		ixgbe_setup_ivar(ixgbe, 0, 0, 0);
5090 		ixgbe_setup_ivar(ixgbe, 0, 1, 1);
5091 		return;
5092 	}
5093 
5094 	/*
5095 	 * For MSI-X interrupt, "Other" is always on vector[0].
5096 	 */
5097 	ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1);
5098 
5099 	/*
5100 	 * For each interrupt vector, populate the IVAR table
5101 	 */
5102 	for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) {
5103 		vect = &ixgbe->vect_map[v_idx];
5104 
5105 		/*
5106 		 * For each rx ring bit set
5107 		 */
5108 		r_idx = bt_getlowbit(vect->rx_map, 0,
5109 		    (ixgbe->num_rx_rings - 1));
5110 
5111 		while (r_idx >= 0) {
5112 			hw_index = ixgbe->rx_rings[r_idx].hw_index;
5113 			ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0);
5114 			r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
5115 			    (ixgbe->num_rx_rings - 1));
5116 		}
5117 
5118 		/*
5119 		 * For each tx ring bit set
5120 		 */
5121 		r_idx = bt_getlowbit(vect->tx_map, 0,
5122 		    (ixgbe->num_tx_rings - 1));
5123 
5124 		while (r_idx >= 0) {
5125 			ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1);
5126 			r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
5127 			    (ixgbe->num_tx_rings - 1));
5128 		}
5129 	}
5130 }
5131 
5132 /*
5133  * ixgbe_rem_intr_handlers - Remove the interrupt handlers.
5134  */
5135 static void
5136 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe)
5137 {
5138 	int i;
5139 	int rc;
5140 
5141 	for (i = 0; i < ixgbe->intr_cnt; i++) {
5142 		rc = ddi_intr_remove_handler(ixgbe->htable[i]);
5143 		if (rc != DDI_SUCCESS) {
5144 			IXGBE_DEBUGLOG_1(ixgbe,
5145 			    "Remove intr handler failed: %d", rc);
5146 		}
5147 	}
5148 }
5149 
5150 /*
5151  * ixgbe_rem_intrs - Remove the allocated interrupts.
5152  */
5153 static void
5154 ixgbe_rem_intrs(ixgbe_t *ixgbe)
5155 {
5156 	int i;
5157 	int rc;
5158 
5159 	for (i = 0; i < ixgbe->intr_cnt; i++) {
5160 		rc = ddi_intr_free(ixgbe->htable[i]);
5161 		if (rc != DDI_SUCCESS) {
5162 			IXGBE_DEBUGLOG_1(ixgbe,
5163 			    "Free intr failed: %d", rc);
5164 		}
5165 	}
5166 
5167 	kmem_free(ixgbe->htable, ixgbe->intr_size);
5168 	ixgbe->htable = NULL;
5169 }
5170 
5171 /*
5172  * ixgbe_enable_intrs - Enable all the ddi interrupts.
5173  */
5174 static int
5175 ixgbe_enable_intrs(ixgbe_t *ixgbe)
5176 {
5177 	int i;
5178 	int rc;
5179 
5180 	/*
5181 	 * Enable interrupts
5182 	 */
5183 	if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5184 		/*
5185 		 * Call ddi_intr_block_enable() for MSI
5186 		 */
5187 		rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt);
5188 		if (rc != DDI_SUCCESS) {
5189 			ixgbe_log(ixgbe,
5190 			    "Enable block intr failed: %d", rc);
5191 			return (IXGBE_FAILURE);
5192 		}
5193 	} else {
5194 		/*
5195 		 * Call ddi_intr_enable() for Legacy/MSI non block enable
5196 		 */
5197 		for (i = 0; i < ixgbe->intr_cnt; i++) {
5198 			rc = ddi_intr_enable(ixgbe->htable[i]);
5199 			if (rc != DDI_SUCCESS) {
5200 				ixgbe_log(ixgbe,
5201 				    "Enable intr failed: %d", rc);
5202 				return (IXGBE_FAILURE);
5203 			}
5204 		}
5205 	}
5206 
5207 	return (IXGBE_SUCCESS);
5208 }
5209 
5210 /*
5211  * ixgbe_disable_intrs - Disable all the interrupts.
5212  */
5213 static int
5214 ixgbe_disable_intrs(ixgbe_t *ixgbe)
5215 {
5216 	int i;
5217 	int rc;
5218 
5219 	/*
5220 	 * Disable all interrupts
5221 	 */
5222 	if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5223 		rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt);
5224 		if (rc != DDI_SUCCESS) {
5225 			ixgbe_log(ixgbe,
5226 			    "Disable block intr failed: %d", rc);
5227 			return (IXGBE_FAILURE);
5228 		}
5229 	} else {
5230 		for (i = 0; i < ixgbe->intr_cnt; i++) {
5231 			rc = ddi_intr_disable(ixgbe->htable[i]);
5232 			if (rc != DDI_SUCCESS) {
5233 				ixgbe_log(ixgbe,
5234 				    "Disable intr failed: %d", rc);
5235 				return (IXGBE_FAILURE);
5236 			}
5237 		}
5238 	}
5239 
5240 	return (IXGBE_SUCCESS);
5241 }
5242 
5243 /*
5244  * ixgbe_get_hw_state - Get and save parameters related to adapter hardware.
5245  */
5246 static void
5247 ixgbe_get_hw_state(ixgbe_t *ixgbe)
5248 {
5249 	struct ixgbe_hw *hw = &ixgbe->hw;
5250 	ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
5251 	boolean_t link_up = B_FALSE;
5252 	uint32_t pcs1g_anlp = 0;
5253 	uint32_t pcs1g_ana = 0;
5254 	boolean_t autoneg = B_FALSE;
5255 
5256 	ASSERT(mutex_owned(&ixgbe->gen_lock));
5257 	ixgbe->param_lp_1000fdx_cap = 0;
5258 	ixgbe->param_lp_100fdx_cap  = 0;
5259 
5260 	/* check for link, don't wait */
5261 	(void) ixgbe_check_link(hw, &speed, &link_up, false);
5262 	pcs1g_ana = IXGBE_READ_REG(hw, IXGBE_PCS1GANA);
5263 
5264 	if (link_up) {
5265 		pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP);
5266 
5267 		ixgbe->param_lp_1000fdx_cap =
5268 		    (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5269 		ixgbe->param_lp_100fdx_cap =
5270 		    (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5271 	}
5272 
5273 	(void) ixgbe_get_link_capabilities(hw, &speed, &autoneg);
5274 
5275 	ixgbe->param_adv_1000fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) &&
5276 	    (speed & IXGBE_LINK_SPEED_1GB_FULL)) ? 1 : 0;
5277 	ixgbe->param_adv_100fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) &&
5278 	    (speed & IXGBE_LINK_SPEED_100_FULL)) ? 1 : 0;
5279 }
5280 
5281 /*
5282  * ixgbe_get_driver_control - Notify that driver is in control of device.
5283  */
5284 static void
5285 ixgbe_get_driver_control(struct ixgbe_hw *hw)
5286 {
5287 	uint32_t ctrl_ext;
5288 
5289 	/*
5290 	 * Notify firmware that driver is in control of device
5291 	 */
5292 	ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5293 	ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
5294 	IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5295 }
5296 
5297 /*
5298  * ixgbe_release_driver_control - Notify that driver is no longer in control
5299  * of device.
5300  */
5301 static void
5302 ixgbe_release_driver_control(struct ixgbe_hw *hw)
5303 {
5304 	uint32_t ctrl_ext;
5305 
5306 	/*
5307 	 * Notify firmware that driver is no longer in control of device
5308 	 */
5309 	ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5310 	ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
5311 	IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5312 }
5313 
5314 /*
5315  * ixgbe_atomic_reserve - Atomic decrease operation.
5316  */
5317 int
5318 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n)
5319 {
5320 	uint32_t oldval;
5321 	uint32_t newval;
5322 
5323 	/*
5324 	 * ATOMICALLY
5325 	 */
5326 	do {
5327 		oldval = *count_p;
5328 		if (oldval < n)
5329 			return (-1);
5330 		newval = oldval - n;
5331 	} while (atomic_cas_32(count_p, oldval, newval) != oldval);
5332 
5333 	return (newval);
5334 }
5335 
5336 /*
5337  * ixgbe_mc_table_itr - Traverse the entries in the multicast table.
5338  */
5339 static uint8_t *
5340 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq)
5341 {
5342 	uint8_t *addr = *upd_ptr;
5343 	uint8_t *new_ptr;
5344 
5345 	_NOTE(ARGUNUSED(hw));
5346 	_NOTE(ARGUNUSED(vmdq));
5347 
5348 	new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
5349 	*upd_ptr = new_ptr;
5350 	return (addr);
5351 }
5352 
5353 /*
5354  * FMA support
5355  */
5356 int
5357 ixgbe_check_acc_handle(ddi_acc_handle_t handle)
5358 {
5359 	ddi_fm_error_t de;
5360 
5361 	ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
5362 	ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
5363 	return (de.fme_status);
5364 }
5365 
5366 int
5367 ixgbe_check_dma_handle(ddi_dma_handle_t handle)
5368 {
5369 	ddi_fm_error_t de;
5370 
5371 	ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
5372 	return (de.fme_status);
5373 }
5374 
5375 /*
5376  * ixgbe_fm_error_cb - The IO fault service error handling callback function.
5377  */
5378 static int
5379 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
5380 {
5381 	_NOTE(ARGUNUSED(impl_data));
5382 	/*
5383 	 * as the driver can always deal with an error in any dma or
5384 	 * access handle, we can just return the fme_status value.
5385 	 */
5386 	pci_ereport_post(dip, err, NULL);
5387 	return (err->fme_status);
5388 }
5389 
5390 static void
5391 ixgbe_fm_init(ixgbe_t *ixgbe)
5392 {
5393 	ddi_iblock_cookie_t iblk;
5394 	int fma_dma_flag;
5395 
5396 	/*
5397 	 * Only register with IO Fault Services if we have some capability
5398 	 */
5399 	if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
5400 		ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
5401 	} else {
5402 		ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
5403 	}
5404 
5405 	if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
5406 		fma_dma_flag = 1;
5407 	} else {
5408 		fma_dma_flag = 0;
5409 	}
5410 
5411 	ixgbe_set_fma_flags(fma_dma_flag);
5412 
5413 	if (ixgbe->fm_capabilities) {
5414 
5415 		/*
5416 		 * Register capabilities with IO Fault Services
5417 		 */
5418 		ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk);
5419 
5420 		/*
5421 		 * Initialize pci ereport capabilities if ereport capable
5422 		 */
5423 		if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
5424 		    DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5425 			pci_ereport_setup(ixgbe->dip);
5426 
5427 		/*
5428 		 * Register error callback if error callback capable
5429 		 */
5430 		if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5431 			ddi_fm_handler_register(ixgbe->dip,
5432 			    ixgbe_fm_error_cb, (void*) ixgbe);
5433 	}
5434 }
5435 
5436 static void
5437 ixgbe_fm_fini(ixgbe_t *ixgbe)
5438 {
5439 	/*
5440 	 * Only unregister FMA capabilities if they are registered
5441 	 */
5442 	if (ixgbe->fm_capabilities) {
5443 
5444 		/*
5445 		 * Release any resources allocated by pci_ereport_setup()
5446 		 */
5447 		if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
5448 		    DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5449 			pci_ereport_teardown(ixgbe->dip);
5450 
5451 		/*
5452 		 * Un-register error callback if error callback capable
5453 		 */
5454 		if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5455 			ddi_fm_handler_unregister(ixgbe->dip);
5456 
5457 		/*
5458 		 * Unregister from IO Fault Service
5459 		 */
5460 		ddi_fm_fini(ixgbe->dip);
5461 	}
5462 }
5463 
5464 void
5465 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail)
5466 {
5467 	uint64_t ena;
5468 	char buf[FM_MAX_CLASS];
5469 
5470 	(void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
5471 	ena = fm_ena_generate(0, FM_ENA_FMT1);
5472 	if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) {
5473 		ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP,
5474 		    FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
5475 	}
5476 }
5477 
5478 static int
5479 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num)
5480 {
5481 	ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh;
5482 
5483 	mutex_enter(&rx_ring->rx_lock);
5484 	rx_ring->ring_gen_num = mr_gen_num;
5485 	mutex_exit(&rx_ring->rx_lock);
5486 	return (0);
5487 }
5488 
5489 /*
5490  * Get the global ring index by a ring index within a group.
5491  */
5492 static int
5493 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex)
5494 {
5495 	ixgbe_rx_ring_t *rx_ring;
5496 	int i;
5497 
5498 	for (i = 0; i < ixgbe->num_rx_rings; i++) {
5499 		rx_ring = &ixgbe->rx_rings[i];
5500 		if (rx_ring->group_index == gindex)
5501 			rindex--;
5502 		if (rindex < 0)
5503 			return (i);
5504 	}
5505 
5506 	return (-1);
5507 }
5508 
5509 /*
5510  * Callback funtion for MAC layer to register all rings.
5511  */
5512 /* ARGSUSED */
5513 void
5514 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index,
5515     const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh)
5516 {
5517 	ixgbe_t *ixgbe = (ixgbe_t *)arg;
5518 	mac_intr_t *mintr = &infop->mri_intr;
5519 
5520 	switch (rtype) {
5521 	case MAC_RING_TYPE_RX: {
5522 		/*
5523 		 * 'index' is the ring index within the group.
5524 		 * Need to get the global ring index by searching in groups.
5525 		 */
5526 		int global_ring_index = ixgbe_get_rx_ring_index(
5527 		    ixgbe, group_index, ring_index);
5528 
5529 		ASSERT(global_ring_index >= 0);
5530 
5531 		ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index];
5532 		rx_ring->ring_handle = rh;
5533 
5534 		infop->mri_driver = (mac_ring_driver_t)rx_ring;
5535 		infop->mri_start = ixgbe_ring_start;
5536 		infop->mri_stop = NULL;
5537 		infop->mri_poll = ixgbe_ring_rx_poll;
5538 		infop->mri_stat = ixgbe_rx_ring_stat;
5539 
5540 		mintr->mi_handle = (mac_intr_handle_t)rx_ring;
5541 		mintr->mi_enable = ixgbe_rx_ring_intr_enable;
5542 		mintr->mi_disable = ixgbe_rx_ring_intr_disable;
5543 		if (ixgbe->intr_type &
5544 		    (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
5545 			mintr->mi_ddi_handle =
5546 			    ixgbe->htable[rx_ring->intr_vector];
5547 		}
5548 
5549 		break;
5550 	}
5551 	case MAC_RING_TYPE_TX: {
5552 		ASSERT(group_index == -1);
5553 		ASSERT(ring_index < ixgbe->num_tx_rings);
5554 
5555 		ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index];
5556 		tx_ring->ring_handle = rh;
5557 
5558 		infop->mri_driver = (mac_ring_driver_t)tx_ring;
5559 		infop->mri_start = NULL;
5560 		infop->mri_stop = NULL;
5561 		infop->mri_tx = ixgbe_ring_tx;
5562 		infop->mri_stat = ixgbe_tx_ring_stat;
5563 		if (ixgbe->intr_type &
5564 		    (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
5565 			mintr->mi_ddi_handle =
5566 			    ixgbe->htable[tx_ring->intr_vector];
5567 		}
5568 		break;
5569 	}
5570 	default:
5571 		break;
5572 	}
5573 }
5574 
5575 /*
5576  * Callback funtion for MAC layer to register all groups.
5577  */
5578 void
5579 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index,
5580     mac_group_info_t *infop, mac_group_handle_t gh)
5581 {
5582 	ixgbe_t *ixgbe = (ixgbe_t *)arg;
5583 
5584 	switch (rtype) {
5585 	case MAC_RING_TYPE_RX: {
5586 		ixgbe_rx_group_t *rx_group;
5587 
5588 		rx_group = &ixgbe->rx_groups[index];
5589 		rx_group->group_handle = gh;
5590 
5591 		infop->mgi_driver = (mac_group_driver_t)rx_group;
5592 		infop->mgi_start = NULL;
5593 		infop->mgi_stop = NULL;
5594 		infop->mgi_addmac = ixgbe_addmac;
5595 		infop->mgi_remmac = ixgbe_remmac;
5596 		infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups);
5597 
5598 		break;
5599 	}
5600 	case MAC_RING_TYPE_TX:
5601 		break;
5602 	default:
5603 		break;
5604 	}
5605 }
5606 
5607 /*
5608  * Enable interrupt on the specificed rx ring.
5609  */
5610 int
5611 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh)
5612 {
5613 	ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
5614 	ixgbe_t *ixgbe = rx_ring->ixgbe;
5615 	int r_idx = rx_ring->index;
5616 	int hw_r_idx = rx_ring->hw_index;
5617 	int v_idx = rx_ring->intr_vector;
5618 
5619 	mutex_enter(&ixgbe->gen_lock);
5620 	if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
5621 		mutex_exit(&ixgbe->gen_lock);
5622 		/*
5623 		 * Simply return 0.
5624 		 * Interrupts are being adjusted. ixgbe_intr_adjust()
5625 		 * will eventually re-enable the interrupt when it's
5626 		 * done with the adjustment.
5627 		 */
5628 		return (0);
5629 	}
5630 
5631 	/*
5632 	 * To enable interrupt by setting the VAL bit of given interrupt
5633 	 * vector allocation register (IVAR).
5634 	 */
5635 	ixgbe_enable_ivar(ixgbe, hw_r_idx, 0);
5636 
5637 	BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
5638 
5639 	/*
5640 	 * Trigger a Rx interrupt on this ring
5641 	 */
5642 	IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx));
5643 	IXGBE_WRITE_FLUSH(&ixgbe->hw);
5644 
5645 	mutex_exit(&ixgbe->gen_lock);
5646 
5647 	return (0);
5648 }
5649 
5650 /*
5651  * Disable interrupt on the specificed rx ring.
5652  */
5653 int
5654 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh)
5655 {
5656 	ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
5657 	ixgbe_t *ixgbe = rx_ring->ixgbe;
5658 	int r_idx = rx_ring->index;
5659 	int hw_r_idx = rx_ring->hw_index;
5660 	int v_idx = rx_ring->intr_vector;
5661 
5662 	mutex_enter(&ixgbe->gen_lock);
5663 	if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
5664 		mutex_exit(&ixgbe->gen_lock);
5665 		/*
5666 		 * Simply return 0.
5667 		 * In the rare case where an interrupt is being
5668 		 * disabled while interrupts are being adjusted,
5669 		 * we don't fail the operation. No interrupts will
5670 		 * be generated while they are adjusted, and
5671 		 * ixgbe_intr_adjust() will cause the interrupts
5672 		 * to be re-enabled once it completes. Note that
5673 		 * in this case, packets may be delivered to the
5674 		 * stack via interrupts before xgbe_rx_ring_intr_enable()
5675 		 * is called again. This is acceptable since interrupt
5676 		 * adjustment is infrequent, and the stack will be
5677 		 * able to handle these packets.
5678 		 */
5679 		return (0);
5680 	}
5681 
5682 	/*
5683 	 * To disable interrupt by clearing the VAL bit of given interrupt
5684 	 * vector allocation register (IVAR).
5685 	 */
5686 	ixgbe_disable_ivar(ixgbe, hw_r_idx, 0);
5687 
5688 	BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx);
5689 
5690 	mutex_exit(&ixgbe->gen_lock);
5691 
5692 	return (0);
5693 }
5694 
5695 /*
5696  * Add a mac address.
5697  */
5698 static int
5699 ixgbe_addmac(void *arg, const uint8_t *mac_addr)
5700 {
5701 	ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
5702 	ixgbe_t *ixgbe = rx_group->ixgbe;
5703 	struct ixgbe_hw *hw = &ixgbe->hw;
5704 	int slot, i;
5705 
5706 	mutex_enter(&ixgbe->gen_lock);
5707 
5708 	if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
5709 		mutex_exit(&ixgbe->gen_lock);
5710 		return (ECANCELED);
5711 	}
5712 
5713 	if (ixgbe->unicst_avail == 0) {
5714 		/* no slots available */
5715 		mutex_exit(&ixgbe->gen_lock);
5716 		return (ENOSPC);
5717 	}
5718 
5719 	/*
5720 	 * The first ixgbe->num_rx_groups slots are reserved for each respective
5721 	 * group. The rest slots are shared by all groups. While adding a
5722 	 * MAC address, reserved slots are firstly checked then the shared
5723 	 * slots are searched.
5724 	 */
5725 	slot = -1;
5726 	if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) {
5727 		for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) {
5728 			if (ixgbe->unicst_addr[i].mac.set == 0) {
5729 				slot = i;
5730 				break;
5731 			}
5732 		}
5733 	} else {
5734 		slot = rx_group->index;
5735 	}
5736 
5737 	if (slot == -1) {
5738 		/* no slots available */
5739 		mutex_exit(&ixgbe->gen_lock);
5740 		return (ENOSPC);
5741 	}
5742 
5743 	bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
5744 	(void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr,
5745 	    rx_group->index, IXGBE_RAH_AV);
5746 	ixgbe->unicst_addr[slot].mac.set = 1;
5747 	ixgbe->unicst_addr[slot].mac.group_index = rx_group->index;
5748 	ixgbe->unicst_avail--;
5749 
5750 	mutex_exit(&ixgbe->gen_lock);
5751 
5752 	return (0);
5753 }
5754 
5755 /*
5756  * Remove a mac address.
5757  */
5758 static int
5759 ixgbe_remmac(void *arg, const uint8_t *mac_addr)
5760 {
5761 	ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
5762 	ixgbe_t *ixgbe = rx_group->ixgbe;
5763 	struct ixgbe_hw *hw = &ixgbe->hw;
5764 	int slot;
5765 
5766 	mutex_enter(&ixgbe->gen_lock);
5767 
5768 	if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
5769 		mutex_exit(&ixgbe->gen_lock);
5770 		return (ECANCELED);
5771 	}
5772 
5773 	slot = ixgbe_unicst_find(ixgbe, mac_addr);
5774 	if (slot == -1) {
5775 		mutex_exit(&ixgbe->gen_lock);
5776 		return (EINVAL);
5777 	}
5778 
5779 	if (ixgbe->unicst_addr[slot].mac.set == 0) {
5780 		mutex_exit(&ixgbe->gen_lock);
5781 		return (EINVAL);
5782 	}
5783 
5784 	bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
5785 	(void) ixgbe_clear_rar(hw, slot);
5786 	ixgbe->unicst_addr[slot].mac.set = 0;
5787 	ixgbe->unicst_avail++;
5788 
5789 	mutex_exit(&ixgbe->gen_lock);
5790 
5791 	return (0);
5792 }
5793