xref: /illumos-gate/usr/src/uts/common/io/igb/igb_main.c (revision 0250c53ad267726f2438e3c6556199a0bbf588a2)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2007-2012 Intel Corporation. All rights reserved.
24  */
25 
26 /*
27  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28  * Copyright 2013, Nexenta Systems, Inc. All rights reserved.
29  * Copyright 2016 Joyent, Inc.
30  * Copyright 2020 Oxide Computer Company
31  */
32 
33 #include "igb_sw.h"
34 
35 static char ident[] = "Intel 1Gb Ethernet";
36 static char igb_version[] = "igb 2.3.8-ish";
37 
38 /*
39  * Local function protoypes
40  */
41 static int igb_register_mac(igb_t *);
42 static int igb_identify_hardware(igb_t *);
43 static int igb_regs_map(igb_t *);
44 static void igb_init_properties(igb_t *);
45 static int igb_init_driver_settings(igb_t *);
46 static void igb_init_locks(igb_t *);
47 static void igb_destroy_locks(igb_t *);
48 static int igb_init_mac_address(igb_t *);
49 static int igb_init(igb_t *);
50 static int igb_init_adapter(igb_t *);
51 static void igb_stop_adapter(igb_t *);
52 static int igb_reset(igb_t *);
53 static void igb_tx_clean(igb_t *);
54 static boolean_t igb_tx_drain(igb_t *);
55 static boolean_t igb_rx_drain(igb_t *);
56 static int igb_alloc_rings(igb_t *);
57 static int igb_alloc_rx_data(igb_t *);
58 static void igb_free_rx_data(igb_t *);
59 static void igb_free_rings(igb_t *);
60 static void igb_setup_rings(igb_t *);
61 static void igb_setup_rx(igb_t *);
62 static void igb_setup_tx(igb_t *);
63 static void igb_setup_rx_ring(igb_rx_ring_t *);
64 static void igb_setup_tx_ring(igb_tx_ring_t *);
65 static void igb_setup_rss(igb_t *);
66 static void igb_setup_mac_rss_classify(igb_t *);
67 static void igb_setup_mac_classify(igb_t *);
68 static void igb_init_unicst(igb_t *);
69 static void igb_setup_multicst(igb_t *);
70 static void igb_get_phy_state(igb_t *);
71 static void igb_param_sync(igb_t *);
72 static void igb_get_conf(igb_t *);
73 static int igb_get_prop(igb_t *, char *, int, int, int);
74 static boolean_t igb_is_link_up(igb_t *);
75 static boolean_t igb_link_check(igb_t *);
76 static void igb_local_timer(void *);
77 static void igb_link_timer(void *);
78 static void igb_arm_watchdog_timer(igb_t *);
79 static void igb_start_watchdog_timer(igb_t *);
80 static void igb_restart_watchdog_timer(igb_t *);
81 static void igb_stop_watchdog_timer(igb_t *);
82 static void igb_start_link_timer(igb_t *);
83 static void igb_stop_link_timer(igb_t *);
84 static void igb_disable_adapter_interrupts(igb_t *);
85 static void igb_enable_adapter_interrupts_82575(igb_t *);
86 static void igb_enable_adapter_interrupts_82576(igb_t *);
87 static void igb_enable_adapter_interrupts_82580(igb_t *);
88 static boolean_t is_valid_mac_addr(uint8_t *);
89 static boolean_t igb_stall_check(igb_t *);
90 static boolean_t igb_set_loopback_mode(igb_t *, uint32_t);
91 static void igb_set_external_loopback(igb_t *);
92 static void igb_set_internal_phy_loopback(igb_t *);
93 static void igb_set_internal_serdes_loopback(igb_t *);
94 static boolean_t igb_find_mac_address(igb_t *);
95 static int igb_alloc_intrs(igb_t *);
96 static int igb_alloc_intr_handles(igb_t *, int);
97 static int igb_add_intr_handlers(igb_t *);
98 static void igb_rem_intr_handlers(igb_t *);
99 static void igb_rem_intrs(igb_t *);
100 static int igb_enable_intrs(igb_t *);
101 static int igb_disable_intrs(igb_t *);
102 static void igb_setup_msix_82575(igb_t *);
103 static void igb_setup_msix_82576(igb_t *);
104 static void igb_setup_msix_82580(igb_t *);
105 static uint_t igb_intr_legacy(void *, void *);
106 static uint_t igb_intr_msi(void *, void *);
107 static uint_t igb_intr_rx(void *, void *);
108 static uint_t igb_intr_tx(void *, void *);
109 static uint_t igb_intr_tx_other(void *, void *);
110 static void igb_intr_rx_work(igb_rx_ring_t *);
111 static void igb_intr_tx_work(igb_tx_ring_t *);
112 static void igb_intr_link_work(igb_t *);
113 static void igb_get_driver_control(struct e1000_hw *);
114 static void igb_release_driver_control(struct e1000_hw *);
115 
116 static int igb_attach(dev_info_t *, ddi_attach_cmd_t);
117 static int igb_detach(dev_info_t *, ddi_detach_cmd_t);
118 static int igb_resume(dev_info_t *);
119 static int igb_suspend(dev_info_t *);
120 static int igb_quiesce(dev_info_t *);
121 static void igb_unconfigure(dev_info_t *, igb_t *);
122 static int igb_fm_error_cb(dev_info_t *, ddi_fm_error_t *,
123     const void *);
124 static void igb_fm_init(igb_t *);
125 static void igb_fm_fini(igb_t *);
126 static void igb_release_multicast(igb_t *);
127 static int igb_ufm_fill_image(ddi_ufm_handle_t *, void *arg, uint_t,
128     ddi_ufm_image_t *);
129 static int igb_ufm_fill_slot(ddi_ufm_handle_t *, void *, uint_t, uint_t,
130     ddi_ufm_slot_t *);
131 static int igb_ufm_getcaps(ddi_ufm_handle_t *, void *, ddi_ufm_cap_t *);
132 static int igb_ufm_readimg(ddi_ufm_handle_t *, void *, uint_t, uint_t,
133     uint64_t, uint64_t, void *, uint64_t *);
134 
135 char *igb_priv_props[] = {
136 	"_eee_support",
137 	"_tx_copy_thresh",
138 	"_tx_recycle_thresh",
139 	"_tx_overload_thresh",
140 	"_tx_resched_thresh",
141 	"_rx_copy_thresh",
142 	"_rx_limit_per_intr",
143 	"_intr_throttling",
144 	"_adv_pause_cap",
145 	"_adv_asym_pause_cap",
146 	NULL
147 };
148 
149 static struct cb_ops igb_cb_ops = {
150 	nulldev,		/* cb_open */
151 	nulldev,		/* cb_close */
152 	nodev,			/* cb_strategy */
153 	nodev,			/* cb_print */
154 	nodev,			/* cb_dump */
155 	nodev,			/* cb_read */
156 	nodev,			/* cb_write */
157 	nodev,			/* cb_ioctl */
158 	nodev,			/* cb_devmap */
159 	nodev,			/* cb_mmap */
160 	nodev,			/* cb_segmap */
161 	nochpoll,		/* cb_chpoll */
162 	ddi_prop_op,		/* cb_prop_op */
163 	NULL,			/* cb_stream */
164 	D_MP | D_HOTPLUG,	/* cb_flag */
165 	CB_REV,			/* cb_rev */
166 	nodev,			/* cb_aread */
167 	nodev			/* cb_awrite */
168 };
169 
170 static struct dev_ops igb_dev_ops = {
171 	DEVO_REV,		/* devo_rev */
172 	0,			/* devo_refcnt */
173 	NULL,			/* devo_getinfo */
174 	nulldev,		/* devo_identify */
175 	nulldev,		/* devo_probe */
176 	igb_attach,		/* devo_attach */
177 	igb_detach,		/* devo_detach */
178 	nodev,			/* devo_reset */
179 	&igb_cb_ops,		/* devo_cb_ops */
180 	NULL,			/* devo_bus_ops */
181 	ddi_power,		/* devo_power */
182 	igb_quiesce,	/* devo_quiesce */
183 };
184 
185 static struct modldrv igb_modldrv = {
186 	&mod_driverops,		/* Type of module.  This one is a driver */
187 	ident,			/* Discription string */
188 	&igb_dev_ops,		/* driver ops */
189 };
190 
191 static struct modlinkage igb_modlinkage = {
192 	MODREV_1, &igb_modldrv, NULL
193 };
194 
195 /* Access attributes for register mapping */
196 ddi_device_acc_attr_t igb_regs_acc_attr = {
197 	DDI_DEVICE_ATTR_V1,
198 	DDI_STRUCTURE_LE_ACC,
199 	DDI_STRICTORDER_ACC,
200 	DDI_FLAGERR_ACC
201 };
202 
203 #define	IGB_M_CALLBACK_FLAGS \
204 	(MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
205 
206 static mac_callbacks_t igb_m_callbacks = {
207 	IGB_M_CALLBACK_FLAGS,
208 	igb_m_stat,
209 	igb_m_start,
210 	igb_m_stop,
211 	igb_m_promisc,
212 	igb_m_multicst,
213 	NULL,
214 	NULL,
215 	NULL,
216 	igb_m_ioctl,
217 	igb_m_getcapab,
218 	NULL,
219 	NULL,
220 	igb_m_setprop,
221 	igb_m_getprop,
222 	igb_m_propinfo
223 };
224 
225 /*
226  * Initialize capabilities of each supported adapter type
227  */
228 static adapter_info_t igb_82575_cap = {
229 	/* limits */
230 	4,		/* maximum number of rx queues */
231 	1,		/* minimum number of rx queues */
232 	4,		/* default number of rx queues */
233 	4,		/* maximum number of tx queues */
234 	1,		/* minimum number of tx queues */
235 	4,		/* default number of tx queues */
236 	65535,		/* maximum interrupt throttle rate */
237 	0,		/* minimum interrupt throttle rate */
238 	200,		/* default interrupt throttle rate */
239 
240 	/* function pointers */
241 	igb_enable_adapter_interrupts_82575,
242 	igb_setup_msix_82575,
243 
244 	/* capabilities */
245 	(IGB_FLAG_HAS_DCA |	/* capability flags */
246 	IGB_FLAG_VMDQ_POOL),
247 
248 	0xffc00000		/* mask for RXDCTL register */
249 };
250 
251 static adapter_info_t igb_82576_cap = {
252 	/* limits */
253 	16,		/* maximum number of rx queues */
254 	1,		/* minimum number of rx queues */
255 	4,		/* default number of rx queues */
256 	16,		/* maximum number of tx queues */
257 	1,		/* minimum number of tx queues */
258 	4,		/* default number of tx queues */
259 	65535,		/* maximum interrupt throttle rate */
260 	0,		/* minimum interrupt throttle rate */
261 	200,		/* default interrupt throttle rate */
262 
263 	/* function pointers */
264 	igb_enable_adapter_interrupts_82576,
265 	igb_setup_msix_82576,
266 
267 	/* capabilities */
268 	(IGB_FLAG_HAS_DCA |	/* capability flags */
269 	IGB_FLAG_VMDQ_POOL |
270 	IGB_FLAG_NEED_CTX_IDX),
271 
272 	0xffe00000		/* mask for RXDCTL register */
273 };
274 
275 static adapter_info_t igb_82580_cap = {
276 	/* limits */
277 	8,		/* maximum number of rx queues */
278 	1,		/* minimum number of rx queues */
279 	4,		/* default number of rx queues */
280 	8,		/* maximum number of tx queues */
281 	1,		/* minimum number of tx queues */
282 	4,		/* default number of tx queues */
283 	65535,		/* maximum interrupt throttle rate */
284 	0,		/* minimum interrupt throttle rate */
285 	200,		/* default interrupt throttle rate */
286 
287 	/* function pointers */
288 	igb_enable_adapter_interrupts_82580,
289 	igb_setup_msix_82580,
290 
291 	/* capabilities */
292 	(IGB_FLAG_HAS_DCA |	/* capability flags */
293 	IGB_FLAG_VMDQ_POOL |
294 	IGB_FLAG_NEED_CTX_IDX),
295 
296 	0xffe00000		/* mask for RXDCTL register */
297 };
298 
299 static adapter_info_t igb_i350_cap = {
300 	/* limits */
301 	8,		/* maximum number of rx queues */
302 	1,		/* minimum number of rx queues */
303 	4,		/* default number of rx queues */
304 	8,		/* maximum number of tx queues */
305 	1,		/* minimum number of tx queues */
306 	4,		/* default number of tx queues */
307 	65535,		/* maximum interrupt throttle rate */
308 	0,		/* minimum interrupt throttle rate */
309 	200,		/* default interrupt throttle rate */
310 
311 	/* function pointers */
312 	igb_enable_adapter_interrupts_82580,
313 	igb_setup_msix_82580,
314 
315 	/* capabilities */
316 	(IGB_FLAG_HAS_DCA |	/* capability flags */
317 	IGB_FLAG_VMDQ_POOL |
318 	IGB_FLAG_NEED_CTX_IDX),
319 
320 	0xffe00000		/* mask for RXDCTL register */
321 };
322 
323 static adapter_info_t igb_i210_cap = {
324 	/* limits */
325 	4,		/* maximum number of rx queues */
326 	1,		/* minimum number of rx queues */
327 	4,		/* default number of rx queues */
328 	4,		/* maximum number of tx queues */
329 	1,		/* minimum number of tx queues */
330 	4,		/* default number of tx queues */
331 	65535,		/* maximum interrupt throttle rate */
332 	0,		/* minimum interrupt throttle rate */
333 	200,		/* default interrupt throttle rate */
334 
335 	/* function pointers */
336 	igb_enable_adapter_interrupts_82580,
337 	igb_setup_msix_82580,
338 
339 	/* capabilities */
340 	(IGB_FLAG_HAS_DCA |	/* capability flags */
341 	IGB_FLAG_VMDQ_POOL |
342 	IGB_FLAG_NEED_CTX_IDX),
343 
344 	0xfff00000		/* mask for RXDCTL register */
345 };
346 
347 static adapter_info_t igb_i354_cap = {
348 	/* limits */
349 	8,		/* maximum number of rx queues */
350 	1,		/* minimum number of rx queues */
351 	4,		/* default number of rx queues */
352 	8,		/* maximum number of tx queues */
353 	1,		/* minimum number of tx queues */
354 	4,		/* default number of tx queues */
355 	65535,		/* maximum interrupt throttle rate */
356 	0,		/* minimum interrupt throttle rate */
357 	200,		/* default interrupt throttle rate */
358 
359 	/* function pointers */
360 	igb_enable_adapter_interrupts_82580,
361 	igb_setup_msix_82580,
362 
363 	/* capabilities */
364 	(IGB_FLAG_HAS_DCA |	/* capability flags */
365 	IGB_FLAG_VMDQ_POOL |
366 	IGB_FLAG_NEED_CTX_IDX),
367 
368 	0xfff00000		/* mask for RXDCTL register */
369 };
370 
371 static ddi_ufm_ops_t igb_ufm_ops = {
372 	.ddi_ufm_op_fill_image = igb_ufm_fill_image,
373 	.ddi_ufm_op_fill_slot = igb_ufm_fill_slot,
374 	.ddi_ufm_op_getcaps = igb_ufm_getcaps,
375 	.ddi_ufm_op_readimg = igb_ufm_readimg
376 };
377 
378 /*
379  * Module Initialization Functions
380  */
381 
382 int
383 _init(void)
384 {
385 	int status;
386 
387 	mac_init_ops(&igb_dev_ops, MODULE_NAME);
388 
389 	status = mod_install(&igb_modlinkage);
390 
391 	if (status != DDI_SUCCESS) {
392 		mac_fini_ops(&igb_dev_ops);
393 	}
394 
395 	return (status);
396 }
397 
398 int
399 _fini(void)
400 {
401 	int status;
402 
403 	status = mod_remove(&igb_modlinkage);
404 
405 	if (status == DDI_SUCCESS) {
406 		mac_fini_ops(&igb_dev_ops);
407 	}
408 
409 	return (status);
410 
411 }
412 
413 int
414 _info(struct modinfo *modinfop)
415 {
416 	int status;
417 
418 	status = mod_info(&igb_modlinkage, modinfop);
419 
420 	return (status);
421 }
422 
423 /*
424  * igb_attach - driver attach
425  *
426  * This function is the device specific initialization entry
427  * point. This entry point is required and must be written.
428  * The DDI_ATTACH command must be provided in the attach entry
429  * point. When attach() is called with cmd set to DDI_ATTACH,
430  * all normal kernel services (such as kmem_alloc(9F)) are
431  * available for use by the driver.
432  *
433  * The attach() function will be called once for each instance
434  * of  the  device  on  the  system with cmd set to DDI_ATTACH.
435  * Until attach() succeeds, the only driver entry points which
436  * may be called are open(9E) and getinfo(9E).
437  */
438 static int
439 igb_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
440 {
441 	igb_t *igb;
442 	struct igb_osdep *osdep;
443 	struct e1000_hw *hw;
444 	int instance;
445 
446 	/*
447 	 * Check the command and perform corresponding operations
448 	 */
449 	switch (cmd) {
450 	default:
451 		return (DDI_FAILURE);
452 
453 	case DDI_RESUME:
454 		return (igb_resume(devinfo));
455 
456 	case DDI_ATTACH:
457 		break;
458 	}
459 
460 	/* Get the device instance */
461 	instance = ddi_get_instance(devinfo);
462 
463 	/* Allocate memory for the instance data structure */
464 	igb = kmem_zalloc(sizeof (igb_t), KM_SLEEP);
465 
466 	igb->dip = devinfo;
467 	igb->instance = instance;
468 
469 	hw = &igb->hw;
470 	osdep = &igb->osdep;
471 	hw->back = osdep;
472 	osdep->igb = igb;
473 
474 	/* Attach the instance pointer to the dev_info data structure */
475 	ddi_set_driver_private(devinfo, igb);
476 
477 
478 	/* Initialize for fma support */
479 	igb->fm_capabilities = igb_get_prop(igb, "fm-capable",
480 	    0, 0x0f,
481 	    DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
482 	    DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
483 	igb_fm_init(igb);
484 	igb->attach_progress |= ATTACH_PROGRESS_FMINIT;
485 
486 	/*
487 	 * Map PCI config space registers
488 	 */
489 	if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
490 		igb_log(igb, IGB_LOG_ERROR, "Failed to map PCI configurations");
491 		goto attach_fail;
492 	}
493 	igb->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
494 
495 	/*
496 	 * Identify the chipset family
497 	 */
498 	if (igb_identify_hardware(igb) != IGB_SUCCESS) {
499 		igb_log(igb, IGB_LOG_ERROR, "Failed to identify hardware");
500 		goto attach_fail;
501 	}
502 
503 	/*
504 	 * Map device registers
505 	 */
506 	if (igb_regs_map(igb) != IGB_SUCCESS) {
507 		igb_log(igb, IGB_LOG_ERROR, "Failed to map device registers");
508 		goto attach_fail;
509 	}
510 	igb->attach_progress |= ATTACH_PROGRESS_REGS_MAP;
511 
512 	/*
513 	 * Initialize driver parameters
514 	 */
515 	igb_init_properties(igb);
516 	igb->attach_progress |= ATTACH_PROGRESS_PROPS;
517 
518 	/*
519 	 * Allocate interrupts
520 	 */
521 	if (igb_alloc_intrs(igb) != IGB_SUCCESS) {
522 		igb_log(igb, IGB_LOG_ERROR, "Failed to allocate interrupts");
523 		goto attach_fail;
524 	}
525 	igb->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
526 
527 	/*
528 	 * Allocate rx/tx rings based on the ring numbers.
529 	 * The actual numbers of rx/tx rings are decided by the number of
530 	 * allocated interrupt vectors, so we should allocate the rings after
531 	 * interrupts are allocated.
532 	 */
533 	if (igb_alloc_rings(igb) != IGB_SUCCESS) {
534 		igb_log(igb, IGB_LOG_ERROR,
535 		    "Failed to allocate rx/tx rings or groups");
536 		goto attach_fail;
537 	}
538 	igb->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS;
539 
540 	/*
541 	 * Add interrupt handlers
542 	 */
543 	if (igb_add_intr_handlers(igb) != IGB_SUCCESS) {
544 		igb_log(igb, IGB_LOG_ERROR, "Failed to add interrupt handlers");
545 		goto attach_fail;
546 	}
547 	igb->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
548 
549 	/*
550 	 * Initialize driver parameters
551 	 */
552 	if (igb_init_driver_settings(igb) != IGB_SUCCESS) {
553 		igb_log(igb, IGB_LOG_ERROR,
554 		    "Failed to initialize driver settings");
555 		goto attach_fail;
556 	}
557 
558 	if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK) {
559 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
560 		goto attach_fail;
561 	}
562 
563 	/*
564 	 * Initialize mutexes for this device.
565 	 * Do this before enabling the interrupt handler and
566 	 * register the softint to avoid the condition where
567 	 * interrupt handler can try using uninitialized mutex
568 	 */
569 	igb_init_locks(igb);
570 	igb->attach_progress |= ATTACH_PROGRESS_LOCKS;
571 
572 	/*
573 	 * Initialize the adapter
574 	 */
575 	if (igb_init(igb) != IGB_SUCCESS) {
576 		igb_log(igb, IGB_LOG_ERROR, "Failed to initialize adapter");
577 		goto attach_fail;
578 	}
579 	igb->attach_progress |= ATTACH_PROGRESS_INIT_ADAPTER;
580 
581 	/*
582 	 * Initialize sensors. This swallows any errors to ensure that access to
583 	 * the network is still available.
584 	 */
585 	igb_init_sensors(igb);
586 
587 	/*
588 	 * Initialize statistics
589 	 */
590 	if (igb_init_stats(igb) != IGB_SUCCESS) {
591 		igb_log(igb, IGB_LOG_ERROR, "Failed to initialize statistics");
592 		goto attach_fail;
593 	}
594 	igb->attach_progress |= ATTACH_PROGRESS_STATS;
595 
596 	/*
597 	 * Register the driver to the MAC
598 	 */
599 	if (igb_register_mac(igb) != IGB_SUCCESS) {
600 		igb_log(igb, IGB_LOG_ERROR, "Failed to register MAC");
601 		goto attach_fail;
602 	}
603 	igb->attach_progress |= ATTACH_PROGRESS_MAC;
604 
605 	/*
606 	 * Now that mutex locks are initialized, and the chip is also
607 	 * initialized, enable interrupts.
608 	 */
609 	if (igb_enable_intrs(igb) != IGB_SUCCESS) {
610 		igb_log(igb, IGB_LOG_ERROR, "Failed to enable DDI interrupts");
611 		goto attach_fail;
612 	}
613 	igb->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
614 
615 	/*
616 	 * Only enable UFM support on function zero of the device as the images
617 	 * are always device wide.
618 	 */
619 	if (igb->hw.bus.func == 0) {
620 		if (ddi_ufm_init(devinfo, DDI_UFM_CURRENT_VERSION, &igb_ufm_ops,
621 		    &igb->igb_ufmh, igb) != 0) {
622 			igb_log(igb, IGB_LOG_ERROR, "Failed to enable DDI UFM "
623 			    "support");
624 			goto attach_fail;
625 		}
626 		igb->attach_progress |= ATTACH_PROGRESS_UFM;
627 		ddi_ufm_update(igb->igb_ufmh);
628 	}
629 
630 	igb_log(igb, IGB_LOG_INFO, "%s", igb_version);
631 	atomic_or_32(&igb->igb_state, IGB_INITIALIZED);
632 
633 	/*
634 	 * Newer models have Energy Efficient Ethernet, let's disable this by
635 	 * default.
636 	 */
637 	if (igb->hw.mac.type == e1000_i350)
638 		(void) e1000_set_eee_i350(&igb->hw, B_FALSE, B_FALSE);
639 	else if (igb->hw.mac.type == e1000_i354)
640 		(void) e1000_set_eee_i354(&igb->hw, B_FALSE, B_FALSE);
641 
642 	return (DDI_SUCCESS);
643 
644 attach_fail:
645 	igb_unconfigure(devinfo, igb);
646 	return (DDI_FAILURE);
647 }
648 
649 /*
650  * igb_detach - driver detach
651  *
652  * The detach() function is the complement of the attach routine.
653  * If cmd is set to DDI_DETACH, detach() is used to remove  the
654  * state  associated  with  a  given  instance of a device node
655  * prior to the removal of that instance from the system.
656  *
657  * The detach() function will be called once for each  instance
658  * of the device for which there has been a successful attach()
659  * once there are no longer  any  opens  on  the  device.
660  *
661  * Interrupts routine are disabled, All memory allocated by this
662  * driver are freed.
663  */
664 static int
665 igb_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
666 {
667 	igb_t *igb;
668 
669 	/*
670 	 * Check detach command
671 	 */
672 	switch (cmd) {
673 	default:
674 		return (DDI_FAILURE);
675 
676 	case DDI_SUSPEND:
677 		return (igb_suspend(devinfo));
678 
679 	case DDI_DETACH:
680 		break;
681 	}
682 
683 
684 	/*
685 	 * Get the pointer to the driver private data structure
686 	 */
687 	igb = (igb_t *)ddi_get_driver_private(devinfo);
688 	if (igb == NULL)
689 		return (DDI_FAILURE);
690 
691 	/*
692 	 * Unregister MAC. If failed, we have to fail the detach
693 	 */
694 	if (mac_unregister(igb->mac_hdl) != 0) {
695 		igb_log(igb, IGB_LOG_ERROR, "Failed to unregister MAC");
696 		return (DDI_FAILURE);
697 	}
698 	igb->attach_progress &= ~ATTACH_PROGRESS_MAC;
699 
700 	/*
701 	 * If the device is still running, it needs to be stopped first.
702 	 * This check is necessary because under some specific circumstances,
703 	 * the detach routine can be called without stopping the interface
704 	 * first.
705 	 */
706 	mutex_enter(&igb->gen_lock);
707 	if (igb->igb_state & IGB_STARTED) {
708 		atomic_and_32(&igb->igb_state, ~IGB_STARTED);
709 		igb_stop(igb, B_TRUE);
710 		mutex_exit(&igb->gen_lock);
711 		/* Disable and stop the watchdog timer */
712 		igb_disable_watchdog_timer(igb);
713 	} else
714 		mutex_exit(&igb->gen_lock);
715 
716 	/*
717 	 * Check if there are still rx buffers held by the upper layer.
718 	 * If so, fail the detach.
719 	 */
720 	if (!igb_rx_drain(igb))
721 		return (DDI_FAILURE);
722 
723 	/*
724 	 * Do the remaining unconfigure routines
725 	 */
726 	igb_unconfigure(devinfo, igb);
727 
728 	return (DDI_SUCCESS);
729 }
730 
731 /*
732  * quiesce(9E) entry point.
733  *
734  * This function is called when the system is single-threaded at high
735  * PIL with preemption disabled. Therefore, this function must not be
736  * blocked.
737  *
738  * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
739  * DDI_FAILURE indicates an error condition and should almost never happen.
740  */
741 static int
742 igb_quiesce(dev_info_t *devinfo)
743 {
744 	igb_t *igb;
745 	struct e1000_hw *hw;
746 
747 	igb = (igb_t *)ddi_get_driver_private(devinfo);
748 
749 	if (igb == NULL)
750 		return (DDI_FAILURE);
751 
752 	hw = &igb->hw;
753 
754 	/*
755 	 * Disable the adapter interrupts
756 	 */
757 	igb_disable_adapter_interrupts(igb);
758 
759 	/* Tell firmware driver is no longer in control */
760 	igb_release_driver_control(hw);
761 
762 	/*
763 	 * Reset the chipset
764 	 */
765 	(void) e1000_reset_hw(hw);
766 
767 	/*
768 	 * Reset PHY if possible
769 	 */
770 	if (e1000_check_reset_block(hw) == E1000_SUCCESS)
771 		(void) e1000_phy_hw_reset(hw);
772 
773 	return (DDI_SUCCESS);
774 }
775 
776 /*
777  * igb_unconfigure - release all resources held by this instance
778  */
779 static void
780 igb_unconfigure(dev_info_t *devinfo, igb_t *igb)
781 {
782 	if (igb->attach_progress & ATTACH_PROGRESS_UFM) {
783 		ddi_ufm_fini(igb->igb_ufmh);
784 	}
785 
786 	/*
787 	 * Disable interrupt
788 	 */
789 	if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
790 		(void) igb_disable_intrs(igb);
791 	}
792 
793 	/*
794 	 * Unregister MAC
795 	 */
796 	if (igb->attach_progress & ATTACH_PROGRESS_MAC) {
797 		(void) mac_unregister(igb->mac_hdl);
798 	}
799 
800 	/*
801 	 * Free statistics
802 	 */
803 	if (igb->attach_progress & ATTACH_PROGRESS_STATS) {
804 		kstat_delete((kstat_t *)igb->igb_ks);
805 	}
806 
807 	/*
808 	 * Remove interrupt handlers
809 	 */
810 	if (igb->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
811 		igb_rem_intr_handlers(igb);
812 	}
813 
814 	/*
815 	 * Remove interrupts
816 	 */
817 	if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
818 		igb_rem_intrs(igb);
819 	}
820 
821 	/*
822 	 * Remove driver properties
823 	 */
824 	if (igb->attach_progress & ATTACH_PROGRESS_PROPS) {
825 		(void) ddi_prop_remove_all(devinfo);
826 	}
827 
828 	/*
829 	 * Stop the adapter
830 	 */
831 	if (igb->attach_progress & ATTACH_PROGRESS_INIT_ADAPTER) {
832 		mutex_enter(&igb->gen_lock);
833 		igb_stop_adapter(igb);
834 		mutex_exit(&igb->gen_lock);
835 		if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK)
836 			ddi_fm_service_impact(igb->dip, DDI_SERVICE_UNAFFECTED);
837 	}
838 
839 	/*
840 	 * Clean up sensors
841 	 */
842 	igb_fini_sensors(igb);
843 
844 	/*
845 	 * Free multicast table
846 	 */
847 	igb_release_multicast(igb);
848 
849 	/*
850 	 * Free register handle
851 	 */
852 	if (igb->attach_progress & ATTACH_PROGRESS_REGS_MAP) {
853 		if (igb->osdep.reg_handle != NULL)
854 			ddi_regs_map_free(&igb->osdep.reg_handle);
855 	}
856 
857 	/*
858 	 * Free PCI config handle
859 	 */
860 	if (igb->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) {
861 		if (igb->osdep.cfg_handle != NULL)
862 			pci_config_teardown(&igb->osdep.cfg_handle);
863 	}
864 
865 	/*
866 	 * Free locks
867 	 */
868 	if (igb->attach_progress & ATTACH_PROGRESS_LOCKS) {
869 		igb_destroy_locks(igb);
870 	}
871 
872 	/*
873 	 * Free the rx/tx rings
874 	 */
875 	if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) {
876 		igb_free_rings(igb);
877 	}
878 
879 	/*
880 	 * Remove FMA
881 	 */
882 	if (igb->attach_progress & ATTACH_PROGRESS_FMINIT) {
883 		igb_fm_fini(igb);
884 	}
885 
886 	/*
887 	 * Free the driver data structure
888 	 */
889 	kmem_free(igb, sizeof (igb_t));
890 
891 	ddi_set_driver_private(devinfo, NULL);
892 }
893 
894 /*
895  * igb_register_mac - Register the driver and its function pointers with
896  * the GLD interface
897  */
898 static int
899 igb_register_mac(igb_t *igb)
900 {
901 	struct e1000_hw *hw = &igb->hw;
902 	mac_register_t *mac;
903 	int status;
904 
905 	if ((mac = mac_alloc(MAC_VERSION)) == NULL)
906 		return (IGB_FAILURE);
907 
908 	mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
909 	mac->m_driver = igb;
910 	mac->m_dip = igb->dip;
911 	mac->m_src_addr = hw->mac.addr;
912 	mac->m_callbacks = &igb_m_callbacks;
913 	mac->m_min_sdu = 0;
914 	mac->m_max_sdu = igb->max_frame_size -
915 	    sizeof (struct ether_vlan_header) - ETHERFCSL;
916 	mac->m_margin = VLAN_TAGSZ;
917 	mac->m_priv_props = igb_priv_props;
918 	mac->m_v12n = MAC_VIRT_LEVEL1;
919 
920 	status = mac_register(mac, &igb->mac_hdl);
921 
922 	mac_free(mac);
923 
924 	return ((status == 0) ? IGB_SUCCESS : IGB_FAILURE);
925 }
926 
927 /*
928  * igb_identify_hardware - Identify the type of the chipset
929  */
930 static int
931 igb_identify_hardware(igb_t *igb)
932 {
933 	struct e1000_hw *hw = &igb->hw;
934 	struct igb_osdep *osdep = &igb->osdep;
935 
936 	/*
937 	 * Get the device id
938 	 */
939 	hw->vendor_id =
940 	    pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID);
941 	hw->device_id =
942 	    pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID);
943 	hw->revision_id =
944 	    pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID);
945 	hw->subsystem_device_id =
946 	    pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID);
947 	hw->subsystem_vendor_id =
948 	    pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID);
949 
950 	/*
951 	 * Set the mac type of the adapter based on the device id
952 	 */
953 	if (e1000_set_mac_type(hw) != E1000_SUCCESS) {
954 		return (IGB_FAILURE);
955 	}
956 
957 	/*
958 	 * Install adapter capabilities based on mac type
959 	 */
960 	switch (hw->mac.type) {
961 	case e1000_82575:
962 		igb->capab = &igb_82575_cap;
963 		break;
964 	case e1000_82576:
965 		igb->capab = &igb_82576_cap;
966 		break;
967 	case e1000_82580:
968 		igb->capab = &igb_82580_cap;
969 		break;
970 	case e1000_i350:
971 		igb->capab = &igb_i350_cap;
972 		break;
973 	case e1000_i210:
974 	case e1000_i211:
975 		igb->capab = &igb_i210_cap;
976 		break;
977 	case e1000_i354:
978 		igb->capab = &igb_i354_cap;
979 		break;
980 	default:
981 		return (IGB_FAILURE);
982 	}
983 
984 	return (IGB_SUCCESS);
985 }
986 
987 /*
988  * igb_regs_map - Map the device registers
989  */
990 static int
991 igb_regs_map(igb_t *igb)
992 {
993 	dev_info_t *devinfo = igb->dip;
994 	struct e1000_hw *hw = &igb->hw;
995 	struct igb_osdep *osdep = &igb->osdep;
996 	off_t mem_size;
997 
998 	/*
999 	 * First get the size of device registers to be mapped.
1000 	 */
1001 	if (ddi_dev_regsize(devinfo, IGB_ADAPTER_REGSET, &mem_size) !=
1002 	    DDI_SUCCESS) {
1003 		return (IGB_FAILURE);
1004 	}
1005 
1006 	/*
1007 	 * Call ddi_regs_map_setup() to map registers
1008 	 */
1009 	if ((ddi_regs_map_setup(devinfo, IGB_ADAPTER_REGSET,
1010 	    (caddr_t *)&hw->hw_addr, 0,
1011 	    mem_size, &igb_regs_acc_attr,
1012 	    &osdep->reg_handle)) != DDI_SUCCESS) {
1013 		return (IGB_FAILURE);
1014 	}
1015 
1016 	return (IGB_SUCCESS);
1017 }
1018 
1019 /*
1020  * igb_init_properties - Initialize driver properties
1021  */
1022 static void
1023 igb_init_properties(igb_t *igb)
1024 {
1025 	/*
1026 	 * Get conf file properties, including link settings
1027 	 * jumbo frames, ring number, descriptor number, etc.
1028 	 */
1029 	igb_get_conf(igb);
1030 }
1031 
1032 /*
1033  * igb_init_driver_settings - Initialize driver settings
1034  *
1035  * The settings include hardware function pointers, bus information,
1036  * rx/tx rings settings, link state, and any other parameters that
1037  * need to be setup during driver initialization.
1038  */
1039 static int
1040 igb_init_driver_settings(igb_t *igb)
1041 {
1042 	struct e1000_hw *hw = &igb->hw;
1043 	igb_rx_ring_t *rx_ring;
1044 	igb_tx_ring_t *tx_ring;
1045 	uint32_t rx_size;
1046 	uint32_t tx_size;
1047 	int i;
1048 
1049 	/*
1050 	 * Initialize chipset specific hardware function pointers
1051 	 */
1052 	if (e1000_setup_init_funcs(hw, B_TRUE) != E1000_SUCCESS) {
1053 		return (IGB_FAILURE);
1054 	}
1055 
1056 	/*
1057 	 * Get bus information
1058 	 */
1059 	if (e1000_get_bus_info(hw) != E1000_SUCCESS) {
1060 		return (IGB_FAILURE);
1061 	}
1062 
1063 	/*
1064 	 * Get the system page size
1065 	 */
1066 	igb->page_size = ddi_ptob(igb->dip, (ulong_t)1);
1067 
1068 	/*
1069 	 * Set rx buffer size
1070 	 * The IP header alignment room is counted in the calculation.
1071 	 * The rx buffer size is in unit of 1K that is required by the
1072 	 * chipset hardware.
1073 	 */
1074 	rx_size = igb->max_frame_size + IPHDR_ALIGN_ROOM;
1075 	igb->rx_buf_size = ((rx_size >> 10) +
1076 	    ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1077 
1078 	/*
1079 	 * Set tx buffer size
1080 	 */
1081 	tx_size = igb->max_frame_size;
1082 	igb->tx_buf_size = ((tx_size >> 10) +
1083 	    ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1084 
1085 	/*
1086 	 * Initialize rx/tx rings parameters
1087 	 */
1088 	for (i = 0; i < igb->num_rx_rings; i++) {
1089 		rx_ring = &igb->rx_rings[i];
1090 		rx_ring->index = i;
1091 		rx_ring->igb = igb;
1092 	}
1093 
1094 	for (i = 0; i < igb->num_tx_rings; i++) {
1095 		tx_ring = &igb->tx_rings[i];
1096 		tx_ring->index = i;
1097 		tx_ring->igb = igb;
1098 		if (igb->tx_head_wb_enable)
1099 			tx_ring->tx_recycle = igb_tx_recycle_head_wb;
1100 		else
1101 			tx_ring->tx_recycle = igb_tx_recycle_legacy;
1102 
1103 		tx_ring->ring_size = igb->tx_ring_size;
1104 		tx_ring->free_list_size = igb->tx_ring_size +
1105 		    (igb->tx_ring_size >> 1);
1106 	}
1107 
1108 	/*
1109 	 * Initialize values of interrupt throttling rates
1110 	 */
1111 	for (i = 1; i < MAX_NUM_EITR; i++)
1112 		igb->intr_throttling[i] = igb->intr_throttling[0];
1113 
1114 	/*
1115 	 * The initial link state should be "unknown"
1116 	 */
1117 	igb->link_state = LINK_STATE_UNKNOWN;
1118 
1119 	return (IGB_SUCCESS);
1120 }
1121 
1122 /*
1123  * igb_init_locks - Initialize locks
1124  */
1125 static void
1126 igb_init_locks(igb_t *igb)
1127 {
1128 	igb_rx_ring_t *rx_ring;
1129 	igb_tx_ring_t *tx_ring;
1130 	int i;
1131 
1132 	for (i = 0; i < igb->num_rx_rings; i++) {
1133 		rx_ring = &igb->rx_rings[i];
1134 		mutex_init(&rx_ring->rx_lock, NULL,
1135 		    MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1136 	}
1137 
1138 	for (i = 0; i < igb->num_tx_rings; i++) {
1139 		tx_ring = &igb->tx_rings[i];
1140 		mutex_init(&tx_ring->tx_lock, NULL,
1141 		    MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1142 		mutex_init(&tx_ring->recycle_lock, NULL,
1143 		    MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1144 		mutex_init(&tx_ring->tcb_head_lock, NULL,
1145 		    MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1146 		mutex_init(&tx_ring->tcb_tail_lock, NULL,
1147 		    MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1148 	}
1149 
1150 	mutex_init(&igb->gen_lock, NULL,
1151 	    MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1152 
1153 	mutex_init(&igb->watchdog_lock, NULL,
1154 	    MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1155 
1156 	mutex_init(&igb->link_lock, NULL,
1157 	    MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1158 }
1159 
1160 /*
1161  * igb_destroy_locks - Destroy locks
1162  */
1163 static void
1164 igb_destroy_locks(igb_t *igb)
1165 {
1166 	igb_rx_ring_t *rx_ring;
1167 	igb_tx_ring_t *tx_ring;
1168 	int i;
1169 
1170 	for (i = 0; i < igb->num_rx_rings; i++) {
1171 		rx_ring = &igb->rx_rings[i];
1172 		mutex_destroy(&rx_ring->rx_lock);
1173 	}
1174 
1175 	for (i = 0; i < igb->num_tx_rings; i++) {
1176 		tx_ring = &igb->tx_rings[i];
1177 		mutex_destroy(&tx_ring->tx_lock);
1178 		mutex_destroy(&tx_ring->recycle_lock);
1179 		mutex_destroy(&tx_ring->tcb_head_lock);
1180 		mutex_destroy(&tx_ring->tcb_tail_lock);
1181 	}
1182 
1183 	mutex_destroy(&igb->gen_lock);
1184 	mutex_destroy(&igb->watchdog_lock);
1185 	mutex_destroy(&igb->link_lock);
1186 }
1187 
1188 static int
1189 igb_resume(dev_info_t *devinfo)
1190 {
1191 	igb_t *igb;
1192 
1193 	igb = (igb_t *)ddi_get_driver_private(devinfo);
1194 	if (igb == NULL)
1195 		return (DDI_FAILURE);
1196 
1197 	mutex_enter(&igb->gen_lock);
1198 
1199 	/*
1200 	 * Enable interrupts
1201 	 */
1202 	if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
1203 		if (igb_enable_intrs(igb) != IGB_SUCCESS) {
1204 			igb_log(igb, IGB_LOG_ERROR,
1205 			    "Failed to enable DDI interrupts");
1206 			mutex_exit(&igb->gen_lock);
1207 			return (DDI_FAILURE);
1208 		}
1209 	}
1210 
1211 	if (igb->igb_state & IGB_STARTED) {
1212 		if (igb_start(igb, B_FALSE) != IGB_SUCCESS) {
1213 			mutex_exit(&igb->gen_lock);
1214 			return (DDI_FAILURE);
1215 		}
1216 
1217 		/*
1218 		 * Enable and start the watchdog timer
1219 		 */
1220 		igb_enable_watchdog_timer(igb);
1221 	}
1222 
1223 	atomic_and_32(&igb->igb_state, ~IGB_SUSPENDED);
1224 
1225 	mutex_exit(&igb->gen_lock);
1226 
1227 	return (DDI_SUCCESS);
1228 }
1229 
1230 static int
1231 igb_suspend(dev_info_t *devinfo)
1232 {
1233 	igb_t *igb;
1234 
1235 	igb = (igb_t *)ddi_get_driver_private(devinfo);
1236 	if (igb == NULL)
1237 		return (DDI_FAILURE);
1238 
1239 	mutex_enter(&igb->gen_lock);
1240 
1241 	atomic_or_32(&igb->igb_state, IGB_SUSPENDED);
1242 
1243 	/*
1244 	 * Disable interrupts
1245 	 */
1246 	if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
1247 		(void) igb_disable_intrs(igb);
1248 	}
1249 
1250 	if (!(igb->igb_state & IGB_STARTED)) {
1251 		mutex_exit(&igb->gen_lock);
1252 		return (DDI_SUCCESS);
1253 	}
1254 
1255 	igb_stop(igb, B_FALSE);
1256 
1257 	mutex_exit(&igb->gen_lock);
1258 
1259 	/*
1260 	 * Disable and stop the watchdog timer
1261 	 */
1262 	igb_disable_watchdog_timer(igb);
1263 
1264 	return (DDI_SUCCESS);
1265 }
1266 
1267 static int
1268 igb_init(igb_t *igb)
1269 {
1270 	mutex_enter(&igb->gen_lock);
1271 
1272 	/*
1273 	 * Initilize the adapter
1274 	 */
1275 	if (igb_init_adapter(igb) != IGB_SUCCESS) {
1276 		mutex_exit(&igb->gen_lock);
1277 		igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE);
1278 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1279 		return (IGB_FAILURE);
1280 	}
1281 
1282 	mutex_exit(&igb->gen_lock);
1283 
1284 	return (IGB_SUCCESS);
1285 }
1286 
1287 /*
1288  * igb_init_mac_address - Initialize the default MAC address
1289  *
1290  * On success, the MAC address is entered in the igb->hw.mac.addr
1291  * and hw->mac.perm_addr fields and the adapter's RAR(0) receive
1292  * address register.
1293  *
1294  * Important side effects:
1295  * 1. adapter is reset - this is required to put it in a known state.
1296  * 2. all of non-volatile memory (NVM) is read & checksummed - NVM is where
1297  * MAC address and all default settings are stored, so a valid checksum
1298  * is required.
1299  */
1300 static int
1301 igb_init_mac_address(igb_t *igb)
1302 {
1303 	struct e1000_hw *hw = &igb->hw;
1304 
1305 	ASSERT(mutex_owned(&igb->gen_lock));
1306 
1307 	/*
1308 	 * Reset chipset to put the hardware in a known state
1309 	 * before we try to get MAC address from NVM.
1310 	 */
1311 	if (e1000_reset_hw(hw) != E1000_SUCCESS) {
1312 		igb_log(igb, IGB_LOG_ERROR, "Adapter reset failed.");
1313 		goto init_mac_fail;
1314 	}
1315 
1316 	/*
1317 	 * NVM validation
1318 	 */
1319 	if (((igb->hw.mac.type != e1000_i210) &&
1320 	    (igb->hw.mac.type != e1000_i211)) &&
1321 	    (e1000_validate_nvm_checksum(hw) < 0)) {
1322 		/*
1323 		 * Some PCI-E parts fail the first check due to
1324 		 * the link being in sleep state.  Call it again,
1325 		 * if it fails a second time its a real issue.
1326 		 */
1327 		if (e1000_validate_nvm_checksum(hw) < 0) {
1328 			igb_log(igb, IGB_LOG_ERROR,
1329 			    "Invalid NVM checksum. Please contact "
1330 			    "the vendor to update the NVM.");
1331 			goto init_mac_fail;
1332 		}
1333 	}
1334 
1335 	/*
1336 	 * Get the mac address
1337 	 * This function should handle SPARC case correctly.
1338 	 */
1339 	if (!igb_find_mac_address(igb)) {
1340 		igb_log(igb, IGB_LOG_ERROR, "Failed to get the mac address");
1341 		goto init_mac_fail;
1342 	}
1343 
1344 	/* Validate mac address */
1345 	if (!is_valid_mac_addr(hw->mac.addr)) {
1346 		igb_log(igb, IGB_LOG_ERROR, "Invalid mac address");
1347 		goto init_mac_fail;
1348 	}
1349 
1350 	return (IGB_SUCCESS);
1351 
1352 init_mac_fail:
1353 	return (IGB_FAILURE);
1354 }
1355 
1356 /*
1357  * igb_init_adapter - Initialize the adapter
1358  */
1359 static int
1360 igb_init_adapter(igb_t *igb)
1361 {
1362 	struct e1000_hw *hw = &igb->hw;
1363 	uint32_t pba;
1364 	int oemid[2];
1365 	uint16_t nvmword;
1366 	uint32_t hwm;
1367 	uint32_t default_mtu;
1368 	u8 pbanum[E1000_PBANUM_LENGTH];
1369 	char eepromver[5];	/* f.ff */
1370 	int i;
1371 
1372 	ASSERT(mutex_owned(&igb->gen_lock));
1373 
1374 	/*
1375 	 * In order to obtain the default MAC address, this will reset the
1376 	 * adapter and validate the NVM that the address and many other
1377 	 * default settings come from.
1378 	 */
1379 	if (igb_init_mac_address(igb) != IGB_SUCCESS) {
1380 		igb_log(igb, IGB_LOG_ERROR, "Failed to initialize MAC address");
1381 		goto init_adapter_fail;
1382 	}
1383 
1384 	/*
1385 	 * Packet Buffer Allocation (PBA)
1386 	 * Writing PBA sets the receive portion of the buffer
1387 	 * the remainder is used for the transmit buffer.
1388 	 */
1389 	switch (hw->mac.type) {
1390 	case e1000_82575:
1391 		pba = E1000_PBA_32K;
1392 		break;
1393 	case e1000_82576:
1394 		pba = E1000_READ_REG(hw, E1000_RXPBS);
1395 		pba &= E1000_RXPBS_SIZE_MASK_82576;
1396 		break;
1397 	case e1000_82580:
1398 	case e1000_i350:
1399 	case e1000_i354:
1400 		pba = E1000_READ_REG(hw, E1000_RXPBS);
1401 		pba = e1000_rxpbs_adjust_82580(pba);
1402 		break;
1403 	case e1000_i210:
1404 	case e1000_i211:
1405 		pba = E1000_PBA_34K;
1406 	default:
1407 		break;
1408 	}
1409 
1410 	/* Special needs in case of Jumbo frames */
1411 	default_mtu = igb_get_prop(igb, PROP_DEFAULT_MTU,
1412 	    MIN_MTU, MAX_MTU, DEFAULT_MTU);
1413 	if ((hw->mac.type == e1000_82575) && (default_mtu > ETHERMTU)) {
1414 		u32 tx_space, min_tx, min_rx;
1415 		pba = E1000_READ_REG(hw, E1000_PBA);
1416 		tx_space = pba >> 16;
1417 		pba &= 0xffff;
1418 		min_tx = (igb->max_frame_size +
1419 		    sizeof (struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2;
1420 		min_tx = roundup(min_tx, 1024);
1421 		min_tx >>= 10;
1422 		min_rx = igb->max_frame_size;
1423 		min_rx = roundup(min_rx, 1024);
1424 		min_rx >>= 10;
1425 		if (tx_space < min_tx &&
1426 		    ((min_tx - tx_space) < pba)) {
1427 			pba = pba - (min_tx - tx_space);
1428 			/*
1429 			 * if short on rx space, rx wins
1430 			 * and must trump tx adjustment
1431 			 */
1432 			if (pba < min_rx)
1433 				pba = min_rx;
1434 		}
1435 		E1000_WRITE_REG(hw, E1000_PBA, pba);
1436 	}
1437 
1438 	DEBUGOUT1("igb_init: pba=%dK", pba);
1439 
1440 	/*
1441 	 * These parameters control the automatic generation (Tx) and
1442 	 * response (Rx) to Ethernet PAUSE frames.
1443 	 * - High water mark should allow for at least two frames to be
1444 	 *   received after sending an XOFF.
1445 	 * - Low water mark works best when it is very near the high water mark.
1446 	 *   This allows the receiver to restart by sending XON when it has
1447 	 *   drained a bit.
1448 	 */
1449 	hwm = min(((pba << 10) * 9 / 10),
1450 	    ((pba << 10) - 2 * igb->max_frame_size));
1451 
1452 	if (hw->mac.type < e1000_82576) {
1453 		hw->fc.high_water = hwm & 0xFFF8;  /* 8-byte granularity */
1454 		hw->fc.low_water = hw->fc.high_water - 8;
1455 	} else {
1456 		hw->fc.high_water = hwm & 0xFFF0;  /* 16-byte granularity */
1457 		hw->fc.low_water = hw->fc.high_water - 16;
1458 	}
1459 
1460 	hw->fc.pause_time = E1000_FC_PAUSE_TIME;
1461 	hw->fc.send_xon = B_TRUE;
1462 
1463 	(void) e1000_validate_mdi_setting(hw);
1464 
1465 	/*
1466 	 * Reset the chipset hardware the second time to put PBA settings
1467 	 * into effect.
1468 	 */
1469 	if (e1000_reset_hw(hw) != E1000_SUCCESS) {
1470 		igb_log(igb, IGB_LOG_ERROR, "Second reset failed");
1471 		goto init_adapter_fail;
1472 	}
1473 
1474 	/*
1475 	 * Don't wait for auto-negotiation to complete
1476 	 */
1477 	hw->phy.autoneg_wait_to_complete = B_FALSE;
1478 
1479 	/*
1480 	 * Copper options
1481 	 */
1482 	if (hw->phy.media_type == e1000_media_type_copper) {
1483 		hw->phy.mdix = 0;	/* AUTO_ALL_MODES */
1484 		hw->phy.disable_polarity_correction = B_FALSE;
1485 		hw->phy.ms_type = e1000_ms_hw_default; /* E1000_MASTER_SLAVE */
1486 	}
1487 
1488 	/*
1489 	 * Initialize link settings
1490 	 */
1491 	(void) igb_setup_link(igb, B_FALSE);
1492 
1493 	/*
1494 	 * Configure/Initialize hardware
1495 	 */
1496 	if (e1000_init_hw(hw) != E1000_SUCCESS) {
1497 		igb_log(igb, IGB_LOG_ERROR, "Failed to initialize hardware");
1498 		goto init_adapter_fail;
1499 	}
1500 
1501 	/*
1502 	 *  Start the link setup timer
1503 	 */
1504 	igb_start_link_timer(igb);
1505 
1506 	/*
1507 	 * Disable wakeup control by default
1508 	 */
1509 	E1000_WRITE_REG(hw, E1000_WUC, 0);
1510 
1511 	/*
1512 	 * Record phy info in hw struct
1513 	 */
1514 	(void) e1000_get_phy_info(hw);
1515 
1516 	/*
1517 	 * Make sure driver has control
1518 	 */
1519 	igb_get_driver_control(hw);
1520 
1521 	/*
1522 	 * Restore LED settings to the default from EEPROM
1523 	 * to meet the standard for Sun platforms.
1524 	 */
1525 	(void) e1000_cleanup_led(hw);
1526 
1527 	/*
1528 	 * Setup MSI-X interrupts
1529 	 */
1530 	if (igb->intr_type == DDI_INTR_TYPE_MSIX)
1531 		igb->capab->setup_msix(igb);
1532 
1533 	/*
1534 	 * Initialize unicast addresses.
1535 	 */
1536 	igb_init_unicst(igb);
1537 
1538 	/*
1539 	 * Setup and initialize the mctable structures.
1540 	 */
1541 	igb_setup_multicst(igb);
1542 
1543 	/*
1544 	 * Set interrupt throttling rate
1545 	 */
1546 	for (i = 0; i < igb->intr_cnt; i++)
1547 		E1000_WRITE_REG(hw, E1000_EITR(i), igb->intr_throttling[i]);
1548 
1549 	/*
1550 	 * Read identifying information and place in devinfo.
1551 	 */
1552 	nvmword = 0xffff;
1553 	(void) e1000_read_nvm(&igb->hw, NVM_OEM_OFFSET_0, 1, &nvmword);
1554 	oemid[0] = (int)nvmword;
1555 	(void) e1000_read_nvm(&igb->hw, NVM_OEM_OFFSET_1, 1, &nvmword);
1556 	oemid[1] = (int)nvmword;
1557 	(void) ddi_prop_update_int_array(DDI_DEV_T_NONE, igb->dip,
1558 	    "oem-identifier", oemid, 2);
1559 
1560 	pbanum[0] = '\0';
1561 	(void) e1000_read_pba_string(&igb->hw, pbanum, sizeof (pbanum));
1562 	if (*pbanum != '\0') {
1563 		(void) ddi_prop_update_string(DDI_DEV_T_NONE, igb->dip,
1564 		    "printed-board-assembly", (char *)pbanum);
1565 	}
1566 
1567 	nvmword = 0xffff;
1568 	(void) e1000_read_nvm(&igb->hw, NVM_VERSION, 1, &nvmword);
1569 	if ((nvmword & 0xf00) == 0) {
1570 		(void) snprintf(eepromver, sizeof (eepromver), "%x.%x",
1571 		    (nvmword & 0xf000) >> 12, (nvmword & 0xff));
1572 		(void) ddi_prop_update_string(DDI_DEV_T_NONE, igb->dip,
1573 		    "nvm-version", eepromver);
1574 	}
1575 
1576 	/*
1577 	 * Save the state of the phy
1578 	 */
1579 	igb_get_phy_state(igb);
1580 
1581 	igb_param_sync(igb);
1582 
1583 	return (IGB_SUCCESS);
1584 
1585 init_adapter_fail:
1586 	/*
1587 	 * Reset PHY if possible
1588 	 */
1589 	if (e1000_check_reset_block(hw) == E1000_SUCCESS)
1590 		(void) e1000_phy_hw_reset(hw);
1591 
1592 	return (IGB_FAILURE);
1593 }
1594 
1595 /*
1596  * igb_stop_adapter - Stop the adapter
1597  */
1598 static void
1599 igb_stop_adapter(igb_t *igb)
1600 {
1601 	struct e1000_hw *hw = &igb->hw;
1602 
1603 	ASSERT(mutex_owned(&igb->gen_lock));
1604 
1605 	/* Stop the link setup timer */
1606 	igb_stop_link_timer(igb);
1607 
1608 	/* Tell firmware driver is no longer in control */
1609 	igb_release_driver_control(hw);
1610 
1611 	/*
1612 	 * Reset the chipset
1613 	 */
1614 	if (e1000_reset_hw(hw) != E1000_SUCCESS) {
1615 		igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE);
1616 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1617 	}
1618 
1619 	/*
1620 	 * e1000_phy_hw_reset is not needed here, MAC reset above is sufficient
1621 	 */
1622 }
1623 
1624 /*
1625  * igb_reset - Reset the chipset and restart the driver.
1626  *
1627  * It involves stopping and re-starting the chipset,
1628  * and re-configuring the rx/tx rings.
1629  */
1630 static int
1631 igb_reset(igb_t *igb)
1632 {
1633 	int i;
1634 
1635 	mutex_enter(&igb->gen_lock);
1636 
1637 	ASSERT(igb->igb_state & IGB_STARTED);
1638 	atomic_and_32(&igb->igb_state, ~IGB_STARTED);
1639 
1640 	/*
1641 	 * Disable the adapter interrupts to stop any rx/tx activities
1642 	 * before draining pending data and resetting hardware.
1643 	 */
1644 	igb_disable_adapter_interrupts(igb);
1645 
1646 	/*
1647 	 * Drain the pending transmit packets
1648 	 */
1649 	(void) igb_tx_drain(igb);
1650 
1651 	for (i = 0; i < igb->num_rx_rings; i++)
1652 		mutex_enter(&igb->rx_rings[i].rx_lock);
1653 	for (i = 0; i < igb->num_tx_rings; i++)
1654 		mutex_enter(&igb->tx_rings[i].tx_lock);
1655 
1656 	/*
1657 	 * Stop the adapter
1658 	 */
1659 	igb_stop_adapter(igb);
1660 
1661 	/*
1662 	 * Clean the pending tx data/resources
1663 	 */
1664 	igb_tx_clean(igb);
1665 
1666 	/*
1667 	 * Start the adapter
1668 	 */
1669 	if (igb_init_adapter(igb) != IGB_SUCCESS) {
1670 		igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE);
1671 		goto reset_failure;
1672 	}
1673 
1674 	/*
1675 	 * Setup the rx/tx rings
1676 	 */
1677 	igb->tx_ring_init = B_FALSE;
1678 	igb_setup_rings(igb);
1679 
1680 	atomic_and_32(&igb->igb_state, ~(IGB_ERROR | IGB_STALL));
1681 
1682 	/*
1683 	 * Enable adapter interrupts
1684 	 * The interrupts must be enabled after the driver state is START
1685 	 */
1686 	igb->capab->enable_intr(igb);
1687 
1688 	if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK)
1689 		goto reset_failure;
1690 
1691 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK)
1692 		goto reset_failure;
1693 
1694 	for (i = igb->num_tx_rings - 1; i >= 0; i--)
1695 		mutex_exit(&igb->tx_rings[i].tx_lock);
1696 	for (i = igb->num_rx_rings - 1; i >= 0; i--)
1697 		mutex_exit(&igb->rx_rings[i].rx_lock);
1698 
1699 	atomic_or_32(&igb->igb_state, IGB_STARTED);
1700 
1701 	mutex_exit(&igb->gen_lock);
1702 
1703 	return (IGB_SUCCESS);
1704 
1705 reset_failure:
1706 	for (i = igb->num_tx_rings - 1; i >= 0; i--)
1707 		mutex_exit(&igb->tx_rings[i].tx_lock);
1708 	for (i = igb->num_rx_rings - 1; i >= 0; i--)
1709 		mutex_exit(&igb->rx_rings[i].rx_lock);
1710 
1711 	mutex_exit(&igb->gen_lock);
1712 
1713 	ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1714 
1715 	return (IGB_FAILURE);
1716 }
1717 
1718 /*
1719  * igb_tx_clean - Clean the pending transmit packets and DMA resources
1720  */
1721 static void
1722 igb_tx_clean(igb_t *igb)
1723 {
1724 	igb_tx_ring_t *tx_ring;
1725 	tx_control_block_t *tcb;
1726 	link_list_t pending_list;
1727 	uint32_t desc_num;
1728 	int i, j;
1729 
1730 	LINK_LIST_INIT(&pending_list);
1731 
1732 	for (i = 0; i < igb->num_tx_rings; i++) {
1733 		tx_ring = &igb->tx_rings[i];
1734 
1735 		mutex_enter(&tx_ring->recycle_lock);
1736 
1737 		/*
1738 		 * Clean the pending tx data - the pending packets in the
1739 		 * work_list that have no chances to be transmitted again.
1740 		 *
1741 		 * We must ensure the chipset is stopped or the link is down
1742 		 * before cleaning the transmit packets.
1743 		 */
1744 		desc_num = 0;
1745 		for (j = 0; j < tx_ring->ring_size; j++) {
1746 			tcb = tx_ring->work_list[j];
1747 			if (tcb != NULL) {
1748 				desc_num += tcb->desc_num;
1749 
1750 				tx_ring->work_list[j] = NULL;
1751 
1752 				igb_free_tcb(tcb);
1753 
1754 				LIST_PUSH_TAIL(&pending_list, &tcb->link);
1755 			}
1756 		}
1757 
1758 		if (desc_num > 0) {
1759 			atomic_add_32(&tx_ring->tbd_free, desc_num);
1760 			ASSERT(tx_ring->tbd_free == tx_ring->ring_size);
1761 
1762 			/*
1763 			 * Reset the head and tail pointers of the tbd ring;
1764 			 * Reset the head write-back if it is enabled.
1765 			 */
1766 			tx_ring->tbd_head = 0;
1767 			tx_ring->tbd_tail = 0;
1768 			if (igb->tx_head_wb_enable)
1769 				*tx_ring->tbd_head_wb = 0;
1770 
1771 			E1000_WRITE_REG(&igb->hw, E1000_TDH(tx_ring->index), 0);
1772 			E1000_WRITE_REG(&igb->hw, E1000_TDT(tx_ring->index), 0);
1773 		}
1774 
1775 		mutex_exit(&tx_ring->recycle_lock);
1776 
1777 		/*
1778 		 * Add the tx control blocks in the pending list to
1779 		 * the free list.
1780 		 */
1781 		igb_put_free_list(tx_ring, &pending_list);
1782 	}
1783 }
1784 
1785 /*
1786  * igb_tx_drain - Drain the tx rings to allow pending packets to be transmitted
1787  */
1788 static boolean_t
1789 igb_tx_drain(igb_t *igb)
1790 {
1791 	igb_tx_ring_t *tx_ring;
1792 	boolean_t done;
1793 	int i, j;
1794 
1795 	/*
1796 	 * Wait for a specific time to allow pending tx packets
1797 	 * to be transmitted.
1798 	 *
1799 	 * Check the counter tbd_free to see if transmission is done.
1800 	 * No lock protection is needed here.
1801 	 *
1802 	 * Return B_TRUE if all pending packets have been transmitted;
1803 	 * Otherwise return B_FALSE;
1804 	 */
1805 	for (i = 0; i < TX_DRAIN_TIME; i++) {
1806 
1807 		done = B_TRUE;
1808 		for (j = 0; j < igb->num_tx_rings; j++) {
1809 			tx_ring = &igb->tx_rings[j];
1810 			done = done &&
1811 			    (tx_ring->tbd_free == tx_ring->ring_size);
1812 		}
1813 
1814 		if (done)
1815 			break;
1816 
1817 		msec_delay(1);
1818 	}
1819 
1820 	return (done);
1821 }
1822 
1823 /*
1824  * igb_rx_drain - Wait for all rx buffers to be released by upper layer
1825  */
1826 static boolean_t
1827 igb_rx_drain(igb_t *igb)
1828 {
1829 	boolean_t done;
1830 	int i;
1831 
1832 	/*
1833 	 * Polling the rx free list to check if those rx buffers held by
1834 	 * the upper layer are released.
1835 	 *
1836 	 * Check the counter rcb_free to see if all pending buffers are
1837 	 * released. No lock protection is needed here.
1838 	 *
1839 	 * Return B_TRUE if all pending buffers have been released;
1840 	 * Otherwise return B_FALSE;
1841 	 */
1842 	for (i = 0; i < RX_DRAIN_TIME; i++) {
1843 		done = (igb->rcb_pending == 0);
1844 
1845 		if (done)
1846 			break;
1847 
1848 		msec_delay(1);
1849 	}
1850 
1851 	return (done);
1852 }
1853 
1854 /*
1855  * igb_start - Start the driver/chipset
1856  */
1857 int
1858 igb_start(igb_t *igb, boolean_t alloc_buffer)
1859 {
1860 	int i;
1861 
1862 	ASSERT(mutex_owned(&igb->gen_lock));
1863 
1864 	if (alloc_buffer) {
1865 		if (igb_alloc_rx_data(igb) != IGB_SUCCESS) {
1866 			igb_log(igb, IGB_LOG_ERROR,
1867 			    "Failed to allocate software receive rings");
1868 			return (IGB_FAILURE);
1869 		}
1870 
1871 		/* Allocate buffers for all the rx/tx rings */
1872 		if (igb_alloc_dma(igb) != IGB_SUCCESS) {
1873 			igb_log(igb, IGB_LOG_ERROR,
1874 			    "Failed to allocate DMA resource");
1875 			return (IGB_FAILURE);
1876 		}
1877 
1878 		igb->tx_ring_init = B_TRUE;
1879 	} else {
1880 		igb->tx_ring_init = B_FALSE;
1881 	}
1882 
1883 	for (i = 0; i < igb->num_rx_rings; i++)
1884 		mutex_enter(&igb->rx_rings[i].rx_lock);
1885 	for (i = 0; i < igb->num_tx_rings; i++)
1886 		mutex_enter(&igb->tx_rings[i].tx_lock);
1887 
1888 	/*
1889 	 * Start the adapter
1890 	 */
1891 	if ((igb->attach_progress & ATTACH_PROGRESS_INIT_ADAPTER) == 0) {
1892 		if (igb_init_adapter(igb) != IGB_SUCCESS) {
1893 			igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE);
1894 			goto start_failure;
1895 		}
1896 		igb->attach_progress |= ATTACH_PROGRESS_INIT_ADAPTER;
1897 	}
1898 
1899 	/*
1900 	 * Setup the rx/tx rings
1901 	 */
1902 	igb_setup_rings(igb);
1903 
1904 	/*
1905 	 * Enable adapter interrupts
1906 	 * The interrupts must be enabled after the driver state is START
1907 	 */
1908 	igb->capab->enable_intr(igb);
1909 
1910 	if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK)
1911 		goto start_failure;
1912 
1913 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK)
1914 		goto start_failure;
1915 
1916 	if (igb->hw.mac.type == e1000_i350)
1917 		(void) e1000_set_eee_i350(&igb->hw, B_FALSE, B_FALSE);
1918 	else if (igb->hw.mac.type == e1000_i354)
1919 		(void) e1000_set_eee_i354(&igb->hw, B_FALSE, B_FALSE);
1920 
1921 	for (i = igb->num_tx_rings - 1; i >= 0; i--)
1922 		mutex_exit(&igb->tx_rings[i].tx_lock);
1923 	for (i = igb->num_rx_rings - 1; i >= 0; i--)
1924 		mutex_exit(&igb->rx_rings[i].rx_lock);
1925 
1926 	return (IGB_SUCCESS);
1927 
1928 start_failure:
1929 	for (i = igb->num_tx_rings - 1; i >= 0; i--)
1930 		mutex_exit(&igb->tx_rings[i].tx_lock);
1931 	for (i = igb->num_rx_rings - 1; i >= 0; i--)
1932 		mutex_exit(&igb->rx_rings[i].rx_lock);
1933 
1934 	ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1935 
1936 	return (IGB_FAILURE);
1937 }
1938 
1939 /*
1940  * igb_stop - Stop the driver/chipset
1941  */
1942 void
1943 igb_stop(igb_t *igb, boolean_t free_buffer)
1944 {
1945 	int i;
1946 
1947 	ASSERT(mutex_owned(&igb->gen_lock));
1948 
1949 	igb->attach_progress &= ~ATTACH_PROGRESS_INIT_ADAPTER;
1950 
1951 	/*
1952 	 * Disable the adapter interrupts
1953 	 */
1954 	igb_disable_adapter_interrupts(igb);
1955 
1956 	/*
1957 	 * Drain the pending tx packets
1958 	 */
1959 	(void) igb_tx_drain(igb);
1960 
1961 	for (i = 0; i < igb->num_rx_rings; i++)
1962 		mutex_enter(&igb->rx_rings[i].rx_lock);
1963 	for (i = 0; i < igb->num_tx_rings; i++)
1964 		mutex_enter(&igb->tx_rings[i].tx_lock);
1965 
1966 	/*
1967 	 * Stop the adapter
1968 	 */
1969 	igb_stop_adapter(igb);
1970 
1971 	/*
1972 	 * Clean the pending tx data/resources
1973 	 */
1974 	igb_tx_clean(igb);
1975 
1976 	for (i = igb->num_tx_rings - 1; i >= 0; i--)
1977 		mutex_exit(&igb->tx_rings[i].tx_lock);
1978 	for (i = igb->num_rx_rings - 1; i >= 0; i--)
1979 		mutex_exit(&igb->rx_rings[i].rx_lock);
1980 
1981 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK)
1982 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1983 
1984 	if (igb->link_state == LINK_STATE_UP) {
1985 		igb->link_state = LINK_STATE_UNKNOWN;
1986 		mac_link_update(igb->mac_hdl, igb->link_state);
1987 	}
1988 
1989 	if (free_buffer) {
1990 		/*
1991 		 * Release the DMA/memory resources of rx/tx rings
1992 		 */
1993 		igb_free_dma(igb);
1994 		igb_free_rx_data(igb);
1995 	}
1996 }
1997 
1998 /*
1999  * igb_alloc_rings - Allocate memory space for rx/tx rings
2000  */
2001 static int
2002 igb_alloc_rings(igb_t *igb)
2003 {
2004 	/*
2005 	 * Allocate memory space for rx rings
2006 	 */
2007 	igb->rx_rings = kmem_zalloc(
2008 	    sizeof (igb_rx_ring_t) * igb->num_rx_rings,
2009 	    KM_NOSLEEP);
2010 
2011 	if (igb->rx_rings == NULL) {
2012 		return (IGB_FAILURE);
2013 	}
2014 
2015 	/*
2016 	 * Allocate memory space for tx rings
2017 	 */
2018 	igb->tx_rings = kmem_zalloc(
2019 	    sizeof (igb_tx_ring_t) * igb->num_tx_rings,
2020 	    KM_NOSLEEP);
2021 
2022 	if (igb->tx_rings == NULL) {
2023 		kmem_free(igb->rx_rings,
2024 		    sizeof (igb_rx_ring_t) * igb->num_rx_rings);
2025 		igb->rx_rings = NULL;
2026 		return (IGB_FAILURE);
2027 	}
2028 
2029 	/*
2030 	 * Allocate memory space for rx ring groups
2031 	 */
2032 	igb->rx_groups = kmem_zalloc(
2033 	    sizeof (igb_rx_group_t) * igb->num_rx_groups,
2034 	    KM_NOSLEEP);
2035 
2036 	if (igb->rx_groups == NULL) {
2037 		kmem_free(igb->rx_rings,
2038 		    sizeof (igb_rx_ring_t) * igb->num_rx_rings);
2039 		kmem_free(igb->tx_rings,
2040 		    sizeof (igb_tx_ring_t) * igb->num_tx_rings);
2041 		igb->rx_rings = NULL;
2042 		igb->tx_rings = NULL;
2043 		return (IGB_FAILURE);
2044 	}
2045 
2046 	return (IGB_SUCCESS);
2047 }
2048 
2049 /*
2050  * igb_free_rings - Free the memory space of rx/tx rings.
2051  */
2052 static void
2053 igb_free_rings(igb_t *igb)
2054 {
2055 	if (igb->rx_rings != NULL) {
2056 		kmem_free(igb->rx_rings,
2057 		    sizeof (igb_rx_ring_t) * igb->num_rx_rings);
2058 		igb->rx_rings = NULL;
2059 	}
2060 
2061 	if (igb->tx_rings != NULL) {
2062 		kmem_free(igb->tx_rings,
2063 		    sizeof (igb_tx_ring_t) * igb->num_tx_rings);
2064 		igb->tx_rings = NULL;
2065 	}
2066 
2067 	if (igb->rx_groups != NULL) {
2068 		kmem_free(igb->rx_groups,
2069 		    sizeof (igb_rx_group_t) * igb->num_rx_groups);
2070 		igb->rx_groups = NULL;
2071 	}
2072 }
2073 
2074 static int
2075 igb_alloc_rx_data(igb_t *igb)
2076 {
2077 	igb_rx_ring_t *rx_ring;
2078 	int i;
2079 
2080 	for (i = 0; i < igb->num_rx_rings; i++) {
2081 		rx_ring = &igb->rx_rings[i];
2082 		if (igb_alloc_rx_ring_data(rx_ring) != IGB_SUCCESS)
2083 			goto alloc_rx_rings_failure;
2084 	}
2085 	return (IGB_SUCCESS);
2086 
2087 alloc_rx_rings_failure:
2088 	igb_free_rx_data(igb);
2089 	return (IGB_FAILURE);
2090 }
2091 
2092 static void
2093 igb_free_rx_data(igb_t *igb)
2094 {
2095 	igb_rx_ring_t *rx_ring;
2096 	igb_rx_data_t *rx_data;
2097 	int i;
2098 
2099 	for (i = 0; i < igb->num_rx_rings; i++) {
2100 		rx_ring = &igb->rx_rings[i];
2101 
2102 		mutex_enter(&igb->rx_pending_lock);
2103 		rx_data = rx_ring->rx_data;
2104 
2105 		if (rx_data != NULL) {
2106 			rx_data->flag |= IGB_RX_STOPPED;
2107 
2108 			if (rx_data->rcb_pending == 0) {
2109 				igb_free_rx_ring_data(rx_data);
2110 				rx_ring->rx_data = NULL;
2111 			}
2112 		}
2113 
2114 		mutex_exit(&igb->rx_pending_lock);
2115 	}
2116 }
2117 
2118 /*
2119  * igb_setup_rings - Setup rx/tx rings
2120  */
2121 static void
2122 igb_setup_rings(igb_t *igb)
2123 {
2124 	/*
2125 	 * Setup the rx/tx rings, including the following:
2126 	 *
2127 	 * 1. Setup the descriptor ring and the control block buffers;
2128 	 * 2. Initialize necessary registers for receive/transmit;
2129 	 * 3. Initialize software pointers/parameters for receive/transmit;
2130 	 */
2131 	igb_setup_rx(igb);
2132 
2133 	igb_setup_tx(igb);
2134 }
2135 
2136 static void
2137 igb_setup_rx_ring(igb_rx_ring_t *rx_ring)
2138 {
2139 	igb_t *igb = rx_ring->igb;
2140 	igb_rx_data_t *rx_data = rx_ring->rx_data;
2141 	struct e1000_hw *hw = &igb->hw;
2142 	rx_control_block_t *rcb;
2143 	union e1000_adv_rx_desc	*rbd;
2144 	uint32_t size;
2145 	uint32_t buf_low;
2146 	uint32_t buf_high;
2147 	uint32_t rxdctl;
2148 	int i;
2149 
2150 	ASSERT(mutex_owned(&rx_ring->rx_lock));
2151 	ASSERT(mutex_owned(&igb->gen_lock));
2152 
2153 	/*
2154 	 * Initialize descriptor ring with buffer addresses
2155 	 */
2156 	for (i = 0; i < igb->rx_ring_size; i++) {
2157 		rcb = rx_data->work_list[i];
2158 		rbd = &rx_data->rbd_ring[i];
2159 
2160 		rbd->read.pkt_addr = rcb->rx_buf.dma_address;
2161 		rbd->read.hdr_addr = 0;
2162 	}
2163 
2164 	/*
2165 	 * Initialize the base address registers
2166 	 */
2167 	buf_low = (uint32_t)rx_data->rbd_area.dma_address;
2168 	buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32);
2169 	E1000_WRITE_REG(hw, E1000_RDBAH(rx_ring->index), buf_high);
2170 	E1000_WRITE_REG(hw, E1000_RDBAL(rx_ring->index), buf_low);
2171 
2172 	/*
2173 	 * Initialize the length register
2174 	 */
2175 	size = rx_data->ring_size * sizeof (union e1000_adv_rx_desc);
2176 	E1000_WRITE_REG(hw, E1000_RDLEN(rx_ring->index), size);
2177 
2178 	/*
2179 	 * Initialize buffer size & descriptor type
2180 	 */
2181 	E1000_WRITE_REG(hw, E1000_SRRCTL(rx_ring->index),
2182 	    ((igb->rx_buf_size >> E1000_SRRCTL_BSIZEPKT_SHIFT) |
2183 	    E1000_SRRCTL_DESCTYPE_ADV_ONEBUF));
2184 
2185 	/*
2186 	 * Setup the Receive Descriptor Control Register (RXDCTL)
2187 	 */
2188 	rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(rx_ring->index));
2189 	rxdctl &= igb->capab->rxdctl_mask;
2190 	rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2191 	rxdctl |= 16;		/* pthresh */
2192 	rxdctl |= 8 << 8;	/* hthresh */
2193 	rxdctl |= 1 << 16;	/* wthresh */
2194 	E1000_WRITE_REG(hw, E1000_RXDCTL(rx_ring->index), rxdctl);
2195 
2196 	rx_data->rbd_next = 0;
2197 }
2198 
2199 static void
2200 igb_setup_rx(igb_t *igb)
2201 {
2202 	igb_rx_ring_t *rx_ring;
2203 	igb_rx_data_t *rx_data;
2204 	igb_rx_group_t *rx_group;
2205 	struct e1000_hw *hw = &igb->hw;
2206 	uint32_t rctl, rxcsum;
2207 	uint32_t ring_per_group;
2208 	int i;
2209 
2210 	/*
2211 	 * Setup the Receive Control Register (RCTL), and enable the
2212 	 * receiver. The initial configuration is to: enable the receiver,
2213 	 * accept broadcasts, discard bad packets, accept long packets,
2214 	 * disable VLAN filter checking, and set receive buffer size to
2215 	 * 2k.  For 82575, also set the receive descriptor minimum
2216 	 * threshold size to 1/2 the ring.
2217 	 */
2218 	rctl = E1000_READ_REG(hw, E1000_RCTL);
2219 
2220 	/*
2221 	 * Clear the field used for wakeup control.  This driver doesn't do
2222 	 * wakeup but leave this here for completeness.
2223 	 */
2224 	rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2225 	rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2226 
2227 	rctl |= (E1000_RCTL_EN |	/* Enable Receive Unit */
2228 	    E1000_RCTL_BAM |		/* Accept Broadcast Packets */
2229 	    E1000_RCTL_LPE |		/* Large Packet Enable */
2230 					/* Multicast filter offset */
2231 	    (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT) |
2232 	    E1000_RCTL_RDMTS_HALF |	/* rx descriptor threshold */
2233 	    E1000_RCTL_SECRC);		/* Strip Ethernet CRC */
2234 
2235 	for (i = 0; i < igb->num_rx_groups; i++) {
2236 		rx_group = &igb->rx_groups[i];
2237 		rx_group->index = i;
2238 		rx_group->igb = igb;
2239 	}
2240 
2241 	/*
2242 	 * Set up all rx descriptor rings - must be called before receive unit
2243 	 * enabled.
2244 	 */
2245 	ring_per_group = igb->num_rx_rings / igb->num_rx_groups;
2246 	for (i = 0; i < igb->num_rx_rings; i++) {
2247 		rx_ring = &igb->rx_rings[i];
2248 		igb_setup_rx_ring(rx_ring);
2249 
2250 		/*
2251 		 * Map a ring to a group by assigning a group index
2252 		 */
2253 		rx_ring->group_index = i / ring_per_group;
2254 	}
2255 
2256 	/*
2257 	 * Setup the Rx Long Packet Max Length register
2258 	 */
2259 	E1000_WRITE_REG(hw, E1000_RLPML, igb->max_frame_size);
2260 
2261 	/*
2262 	 * Hardware checksum settings
2263 	 */
2264 	if (igb->rx_hcksum_enable) {
2265 		rxcsum =
2266 		    E1000_RXCSUM_TUOFL |	/* TCP/UDP checksum */
2267 		    E1000_RXCSUM_IPOFL;		/* IP checksum */
2268 
2269 		E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
2270 	}
2271 
2272 	/*
2273 	 * Setup classify and RSS for multiple receive queues
2274 	 */
2275 	switch (igb->vmdq_mode) {
2276 	case E1000_VMDQ_OFF:
2277 		/*
2278 		 * One ring group, only RSS is needed when more than
2279 		 * one ring enabled.
2280 		 */
2281 		if (igb->num_rx_rings > 1)
2282 			igb_setup_rss(igb);
2283 		break;
2284 	case E1000_VMDQ_MAC:
2285 		/*
2286 		 * Multiple groups, each group has one ring,
2287 		 * only the MAC classification is needed.
2288 		 */
2289 		igb_setup_mac_classify(igb);
2290 		break;
2291 	case E1000_VMDQ_MAC_RSS:
2292 		/*
2293 		 * Multiple groups and multiple rings, both
2294 		 * MAC classification and RSS are needed.
2295 		 */
2296 		igb_setup_mac_rss_classify(igb);
2297 		break;
2298 	}
2299 
2300 	/*
2301 	 * Enable the receive unit - must be done after all
2302 	 * the rx setup above.
2303 	 */
2304 	E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2305 
2306 	/*
2307 	 * Initialize all adapter ring head & tail pointers - must
2308 	 * be done after receive unit is enabled
2309 	 */
2310 	for (i = 0; i < igb->num_rx_rings; i++) {
2311 		rx_ring = &igb->rx_rings[i];
2312 		rx_data = rx_ring->rx_data;
2313 		E1000_WRITE_REG(hw, E1000_RDH(i), 0);
2314 		E1000_WRITE_REG(hw, E1000_RDT(i), rx_data->ring_size - 1);
2315 	}
2316 
2317 	/*
2318 	 * 82575 with manageability enabled needs a special flush to make
2319 	 * sure the fifos start clean.
2320 	 */
2321 	if ((hw->mac.type == e1000_82575) &&
2322 	    (E1000_READ_REG(hw, E1000_MANC) & E1000_MANC_RCV_TCO_EN)) {
2323 		e1000_rx_fifo_flush_82575(hw);
2324 	}
2325 }
2326 
2327 static void
2328 igb_setup_tx_ring(igb_tx_ring_t *tx_ring)
2329 {
2330 	igb_t *igb = tx_ring->igb;
2331 	struct e1000_hw *hw = &igb->hw;
2332 	uint32_t size;
2333 	uint32_t buf_low;
2334 	uint32_t buf_high;
2335 	uint32_t reg_val;
2336 
2337 	ASSERT(mutex_owned(&tx_ring->tx_lock));
2338 	ASSERT(mutex_owned(&igb->gen_lock));
2339 
2340 
2341 	/*
2342 	 * Initialize the length register
2343 	 */
2344 	size = tx_ring->ring_size * sizeof (union e1000_adv_tx_desc);
2345 	E1000_WRITE_REG(hw, E1000_TDLEN(tx_ring->index), size);
2346 
2347 	/*
2348 	 * Initialize the base address registers
2349 	 */
2350 	buf_low = (uint32_t)tx_ring->tbd_area.dma_address;
2351 	buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32);
2352 	E1000_WRITE_REG(hw, E1000_TDBAL(tx_ring->index), buf_low);
2353 	E1000_WRITE_REG(hw, E1000_TDBAH(tx_ring->index), buf_high);
2354 
2355 	/*
2356 	 * Setup head & tail pointers
2357 	 */
2358 	E1000_WRITE_REG(hw, E1000_TDH(tx_ring->index), 0);
2359 	E1000_WRITE_REG(hw, E1000_TDT(tx_ring->index), 0);
2360 
2361 	/*
2362 	 * Setup head write-back
2363 	 */
2364 	if (igb->tx_head_wb_enable) {
2365 		/*
2366 		 * The memory of the head write-back is allocated using
2367 		 * the extra tbd beyond the tail of the tbd ring.
2368 		 */
2369 		tx_ring->tbd_head_wb = (uint32_t *)
2370 		    ((uintptr_t)tx_ring->tbd_area.address + size);
2371 		*tx_ring->tbd_head_wb = 0;
2372 
2373 		buf_low = (uint32_t)
2374 		    (tx_ring->tbd_area.dma_address + size);
2375 		buf_high = (uint32_t)
2376 		    ((tx_ring->tbd_area.dma_address + size) >> 32);
2377 
2378 		/* Set the head write-back enable bit */
2379 		buf_low |= E1000_TX_HEAD_WB_ENABLE;
2380 
2381 		E1000_WRITE_REG(hw, E1000_TDWBAL(tx_ring->index), buf_low);
2382 		E1000_WRITE_REG(hw, E1000_TDWBAH(tx_ring->index), buf_high);
2383 
2384 		/*
2385 		 * Turn off relaxed ordering for head write back or it will
2386 		 * cause problems with the tx recycling
2387 		 */
2388 		reg_val = E1000_READ_REG(hw,
2389 		    E1000_DCA_TXCTRL(tx_ring->index));
2390 		reg_val &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN;
2391 		E1000_WRITE_REG(hw,
2392 		    E1000_DCA_TXCTRL(tx_ring->index), reg_val);
2393 	} else {
2394 		tx_ring->tbd_head_wb = NULL;
2395 	}
2396 
2397 	tx_ring->tbd_head = 0;
2398 	tx_ring->tbd_tail = 0;
2399 	tx_ring->tbd_free = tx_ring->ring_size;
2400 
2401 	if (igb->tx_ring_init == B_TRUE) {
2402 		tx_ring->tcb_head = 0;
2403 		tx_ring->tcb_tail = 0;
2404 		tx_ring->tcb_free = tx_ring->free_list_size;
2405 	}
2406 
2407 	/*
2408 	 * Enable TXDCTL per queue
2409 	 */
2410 	reg_val = E1000_READ_REG(hw, E1000_TXDCTL(tx_ring->index));
2411 	reg_val |= E1000_TXDCTL_QUEUE_ENABLE;
2412 	E1000_WRITE_REG(hw, E1000_TXDCTL(tx_ring->index), reg_val);
2413 
2414 	/*
2415 	 * Initialize hardware checksum offload settings
2416 	 */
2417 	bzero(&tx_ring->tx_context, sizeof (tx_context_t));
2418 }
2419 
2420 static void
2421 igb_setup_tx(igb_t *igb)
2422 {
2423 	igb_tx_ring_t *tx_ring;
2424 	struct e1000_hw *hw = &igb->hw;
2425 	uint32_t reg_val;
2426 	int i;
2427 
2428 	for (i = 0; i < igb->num_tx_rings; i++) {
2429 		tx_ring = &igb->tx_rings[i];
2430 		igb_setup_tx_ring(tx_ring);
2431 	}
2432 
2433 	/*
2434 	 * Setup the Transmit Control Register (TCTL)
2435 	 */
2436 	reg_val = E1000_READ_REG(hw, E1000_TCTL);
2437 	reg_val &= ~E1000_TCTL_CT;
2438 	reg_val |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2439 	    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2440 
2441 	/* Enable transmits */
2442 	reg_val |= E1000_TCTL_EN;
2443 
2444 	E1000_WRITE_REG(hw, E1000_TCTL, reg_val);
2445 }
2446 
2447 /*
2448  * igb_setup_rss - Setup receive-side scaling feature
2449  */
2450 static void
2451 igb_setup_rss(igb_t *igb)
2452 {
2453 	struct e1000_hw *hw = &igb->hw;
2454 	uint32_t i, mrqc, rxcsum;
2455 	int shift = 0;
2456 	uint32_t random;
2457 	union e1000_reta {
2458 		uint32_t	dword;
2459 		uint8_t		bytes[4];
2460 	} reta;
2461 
2462 	/* Setup the Redirection Table */
2463 	if (hw->mac.type == e1000_82576) {
2464 		shift = 3;
2465 	} else if (hw->mac.type == e1000_82575) {
2466 		shift = 6;
2467 	}
2468 	for (i = 0; i < (32 * 4); i++) {
2469 		reta.bytes[i & 3] = (i % igb->num_rx_rings) << shift;
2470 		if ((i & 3) == 3) {
2471 			E1000_WRITE_REG(hw,
2472 			    (E1000_RETA(0) + (i & ~3)), reta.dword);
2473 		}
2474 	}
2475 
2476 	/* Fill out hash function seeds */
2477 	for (i = 0; i < 10; i++) {
2478 		(void) random_get_pseudo_bytes((uint8_t *)&random,
2479 		    sizeof (uint32_t));
2480 		E1000_WRITE_REG(hw, E1000_RSSRK(i), random);
2481 	}
2482 
2483 	/* Setup the Multiple Receive Queue Control register */
2484 	mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2485 	mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2486 	    E1000_MRQC_RSS_FIELD_IPV4_TCP |
2487 	    E1000_MRQC_RSS_FIELD_IPV6 |
2488 	    E1000_MRQC_RSS_FIELD_IPV6_TCP |
2489 	    E1000_MRQC_RSS_FIELD_IPV4_UDP |
2490 	    E1000_MRQC_RSS_FIELD_IPV6_UDP |
2491 	    E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2492 	    E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2493 
2494 	E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
2495 
2496 	/*
2497 	 * Disable Packet Checksum to enable RSS for multiple receive queues.
2498 	 *
2499 	 * The Packet Checksum is not ethernet CRC. It is another kind of
2500 	 * checksum offloading provided by the 82575 chipset besides the IP
2501 	 * header checksum offloading and the TCP/UDP checksum offloading.
2502 	 * The Packet Checksum is by default computed over the entire packet
2503 	 * from the first byte of the DA through the last byte of the CRC,
2504 	 * including the Ethernet and IP headers.
2505 	 *
2506 	 * It is a hardware limitation that Packet Checksum is mutually
2507 	 * exclusive with RSS.
2508 	 */
2509 	rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
2510 	rxcsum |= E1000_RXCSUM_PCSD;
2511 	E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
2512 }
2513 
2514 /*
2515  * igb_setup_mac_rss_classify - Setup MAC classification and rss
2516  */
2517 static void
2518 igb_setup_mac_rss_classify(igb_t *igb)
2519 {
2520 	struct e1000_hw *hw = &igb->hw;
2521 	uint32_t i, mrqc, vmdctl, rxcsum;
2522 	uint32_t ring_per_group;
2523 	int shift_group0, shift_group1;
2524 	uint32_t random;
2525 	union e1000_reta {
2526 		uint32_t	dword;
2527 		uint8_t		bytes[4];
2528 	} reta;
2529 
2530 	ring_per_group = igb->num_rx_rings / igb->num_rx_groups;
2531 
2532 	/* Setup the Redirection Table, it is shared between two groups */
2533 	shift_group0 = 2;
2534 	shift_group1 = 6;
2535 	for (i = 0; i < (32 * 4); i++) {
2536 		reta.bytes[i & 3] = ((i % ring_per_group) << shift_group0) |
2537 		    ((ring_per_group + (i % ring_per_group)) << shift_group1);
2538 		if ((i & 3) == 3) {
2539 			E1000_WRITE_REG(hw,
2540 			    (E1000_RETA(0) + (i & ~3)), reta.dword);
2541 		}
2542 	}
2543 
2544 	/* Fill out hash function seeds */
2545 	for (i = 0; i < 10; i++) {
2546 		(void) random_get_pseudo_bytes((uint8_t *)&random,
2547 		    sizeof (uint32_t));
2548 		E1000_WRITE_REG(hw, E1000_RSSRK(i), random);
2549 	}
2550 
2551 	/*
2552 	 * Setup the Multiple Receive Queue Control register,
2553 	 * enable VMDq based on packet destination MAC address and RSS.
2554 	 */
2555 	mrqc = E1000_MRQC_ENABLE_VMDQ_MAC_RSS_GROUP;
2556 	mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2557 	    E1000_MRQC_RSS_FIELD_IPV4_TCP |
2558 	    E1000_MRQC_RSS_FIELD_IPV6 |
2559 	    E1000_MRQC_RSS_FIELD_IPV6_TCP |
2560 	    E1000_MRQC_RSS_FIELD_IPV4_UDP |
2561 	    E1000_MRQC_RSS_FIELD_IPV6_UDP |
2562 	    E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2563 	    E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2564 
2565 	E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
2566 
2567 
2568 	/* Define the default group and default queues */
2569 	vmdctl = E1000_VMDQ_MAC_GROUP_DEFAULT_QUEUE;
2570 	E1000_WRITE_REG(hw, E1000_VT_CTL, vmdctl);
2571 
2572 	/*
2573 	 * Disable Packet Checksum to enable RSS for multiple receive queues.
2574 	 *
2575 	 * The Packet Checksum is not ethernet CRC. It is another kind of
2576 	 * checksum offloading provided by the 82575 chipset besides the IP
2577 	 * header checksum offloading and the TCP/UDP checksum offloading.
2578 	 * The Packet Checksum is by default computed over the entire packet
2579 	 * from the first byte of the DA through the last byte of the CRC,
2580 	 * including the Ethernet and IP headers.
2581 	 *
2582 	 * It is a hardware limitation that Packet Checksum is mutually
2583 	 * exclusive with RSS.
2584 	 */
2585 	rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
2586 	rxcsum |= E1000_RXCSUM_PCSD;
2587 	E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
2588 }
2589 
2590 /*
2591  * igb_setup_mac_classify - Setup MAC classification feature
2592  */
2593 static void
2594 igb_setup_mac_classify(igb_t *igb)
2595 {
2596 	struct e1000_hw *hw = &igb->hw;
2597 	uint32_t mrqc, rxcsum;
2598 
2599 	/*
2600 	 * Setup the Multiple Receive Queue Control register,
2601 	 * enable VMDq based on packet destination MAC address.
2602 	 */
2603 	mrqc = E1000_MRQC_ENABLE_VMDQ_MAC_GROUP;
2604 	E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
2605 
2606 	/*
2607 	 * Disable Packet Checksum to enable RSS for multiple receive queues.
2608 	 *
2609 	 * The Packet Checksum is not ethernet CRC. It is another kind of
2610 	 * checksum offloading provided by the 82575 chipset besides the IP
2611 	 * header checksum offloading and the TCP/UDP checksum offloading.
2612 	 * The Packet Checksum is by default computed over the entire packet
2613 	 * from the first byte of the DA through the last byte of the CRC,
2614 	 * including the Ethernet and IP headers.
2615 	 *
2616 	 * It is a hardware limitation that Packet Checksum is mutually
2617 	 * exclusive with RSS.
2618 	 */
2619 	rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
2620 	rxcsum |= E1000_RXCSUM_PCSD;
2621 	E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
2622 
2623 }
2624 
2625 /*
2626  * igb_init_unicst - Initialize the unicast addresses
2627  */
2628 static void
2629 igb_init_unicst(igb_t *igb)
2630 {
2631 	struct e1000_hw *hw = &igb->hw;
2632 	int slot;
2633 
2634 	/*
2635 	 * Here we should consider two situations:
2636 	 *
2637 	 * 1. Chipset is initialized the first time
2638 	 *    Initialize the multiple unicast addresses, and
2639 	 *    save the default MAC address.
2640 	 *
2641 	 * 2. Chipset is reset
2642 	 *    Recover the multiple unicast addresses from the
2643 	 *    software data structure to the RAR registers.
2644 	 */
2645 
2646 	/*
2647 	 * Clear the default MAC address in the RAR0 rgister,
2648 	 * which is loaded from EEPROM when system boot or chipreset,
2649 	 * this will cause the conficts with add_mac/rem_mac entry
2650 	 * points when VMDq is enabled. For this reason, the RAR0
2651 	 * must be cleared for both cases mentioned above.
2652 	 */
2653 	e1000_rar_clear(hw, 0);
2654 
2655 	if (!igb->unicst_init) {
2656 
2657 		/* Initialize the multiple unicast addresses */
2658 		igb->unicst_total = MAX_NUM_UNICAST_ADDRESSES;
2659 		igb->unicst_avail = igb->unicst_total;
2660 
2661 		for (slot = 0; slot < igb->unicst_total; slot++)
2662 			igb->unicst_addr[slot].mac.set = 0;
2663 
2664 		igb->unicst_init = B_TRUE;
2665 	} else {
2666 		/* Re-configure the RAR registers */
2667 		for (slot = 0; slot < igb->unicst_total; slot++) {
2668 			(void) e1000_rar_set_vmdq(hw,
2669 			    igb->unicst_addr[slot].mac.addr,
2670 			    slot, igb->vmdq_mode,
2671 			    igb->unicst_addr[slot].mac.group_index);
2672 		}
2673 	}
2674 }
2675 
2676 /*
2677  * igb_unicst_find - Find the slot for the specified unicast address
2678  */
2679 int
2680 igb_unicst_find(igb_t *igb, const uint8_t *mac_addr)
2681 {
2682 	int slot;
2683 
2684 	ASSERT(mutex_owned(&igb->gen_lock));
2685 
2686 	for (slot = 0; slot < igb->unicst_total; slot++) {
2687 		if (bcmp(igb->unicst_addr[slot].mac.addr,
2688 		    mac_addr, ETHERADDRL) == 0)
2689 			return (slot);
2690 	}
2691 
2692 	return (-1);
2693 }
2694 
2695 /*
2696  * igb_unicst_set - Set the unicast address to the specified slot
2697  */
2698 int
2699 igb_unicst_set(igb_t *igb, const uint8_t *mac_addr,
2700     int slot)
2701 {
2702 	struct e1000_hw *hw = &igb->hw;
2703 
2704 	ASSERT(mutex_owned(&igb->gen_lock));
2705 
2706 	/*
2707 	 * Save the unicast address in the software data structure
2708 	 */
2709 	bcopy(mac_addr, igb->unicst_addr[slot].mac.addr, ETHERADDRL);
2710 
2711 	/*
2712 	 * Set the unicast address to the RAR register
2713 	 */
2714 	(void) e1000_rar_set(hw, (uint8_t *)mac_addr, slot);
2715 
2716 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
2717 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
2718 		return (EIO);
2719 	}
2720 
2721 	return (0);
2722 }
2723 
2724 /*
2725  * igb_multicst_add - Add a multicst address
2726  */
2727 int
2728 igb_multicst_add(igb_t *igb, const uint8_t *multiaddr)
2729 {
2730 	struct ether_addr *new_table;
2731 	size_t new_len;
2732 	size_t old_len;
2733 
2734 	ASSERT(mutex_owned(&igb->gen_lock));
2735 
2736 	if ((multiaddr[0] & 01) == 0) {
2737 		igb_log(igb, IGB_LOG_ERROR, "Illegal multicast address");
2738 		return (EINVAL);
2739 	}
2740 
2741 	if (igb->mcast_count >= igb->mcast_max_num) {
2742 		igb_log(igb, IGB_LOG_ERROR,
2743 		    "Adapter requested more than %d mcast addresses",
2744 		    igb->mcast_max_num);
2745 		return (ENOENT);
2746 	}
2747 
2748 	if (igb->mcast_count == igb->mcast_alloc_count) {
2749 		old_len = igb->mcast_alloc_count *
2750 		    sizeof (struct ether_addr);
2751 		new_len = (igb->mcast_alloc_count + MCAST_ALLOC_COUNT) *
2752 		    sizeof (struct ether_addr);
2753 
2754 		new_table = kmem_alloc(new_len, KM_NOSLEEP);
2755 		if (new_table == NULL) {
2756 			igb_log(igb, IGB_LOG_ERROR,
2757 			    "Not enough memory to alloc mcast table");
2758 			return (ENOMEM);
2759 		}
2760 
2761 		if (igb->mcast_table != NULL) {
2762 			bcopy(igb->mcast_table, new_table, old_len);
2763 			kmem_free(igb->mcast_table, old_len);
2764 		}
2765 		igb->mcast_alloc_count += MCAST_ALLOC_COUNT;
2766 		igb->mcast_table = new_table;
2767 	}
2768 
2769 	bcopy(multiaddr,
2770 	    &igb->mcast_table[igb->mcast_count], ETHERADDRL);
2771 	igb->mcast_count++;
2772 
2773 	/*
2774 	 * Update the multicast table in the hardware
2775 	 */
2776 	igb_setup_multicst(igb);
2777 
2778 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
2779 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
2780 		return (EIO);
2781 	}
2782 
2783 	return (0);
2784 }
2785 
2786 /*
2787  * igb_multicst_remove - Remove a multicst address
2788  */
2789 int
2790 igb_multicst_remove(igb_t *igb, const uint8_t *multiaddr)
2791 {
2792 	struct ether_addr *new_table;
2793 	size_t new_len;
2794 	size_t old_len;
2795 	int i;
2796 
2797 	ASSERT(mutex_owned(&igb->gen_lock));
2798 
2799 	for (i = 0; i < igb->mcast_count; i++) {
2800 		if (bcmp(multiaddr, &igb->mcast_table[i],
2801 		    ETHERADDRL) == 0) {
2802 			for (i++; i < igb->mcast_count; i++) {
2803 				igb->mcast_table[i - 1] =
2804 				    igb->mcast_table[i];
2805 			}
2806 			igb->mcast_count--;
2807 			break;
2808 		}
2809 	}
2810 
2811 	if ((igb->mcast_alloc_count - igb->mcast_count) >
2812 	    MCAST_ALLOC_COUNT) {
2813 		old_len = igb->mcast_alloc_count *
2814 		    sizeof (struct ether_addr);
2815 		new_len = (igb->mcast_alloc_count - MCAST_ALLOC_COUNT) *
2816 		    sizeof (struct ether_addr);
2817 
2818 		new_table = kmem_alloc(new_len, KM_NOSLEEP);
2819 		if (new_table != NULL) {
2820 			bcopy(igb->mcast_table, new_table, new_len);
2821 			kmem_free(igb->mcast_table, old_len);
2822 			igb->mcast_alloc_count -= MCAST_ALLOC_COUNT;
2823 			igb->mcast_table = new_table;
2824 		}
2825 	}
2826 
2827 	/*
2828 	 * Update the multicast table in the hardware
2829 	 */
2830 	igb_setup_multicst(igb);
2831 
2832 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
2833 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
2834 		return (EIO);
2835 	}
2836 
2837 	return (0);
2838 }
2839 
2840 static void
2841 igb_release_multicast(igb_t *igb)
2842 {
2843 	if (igb->mcast_table != NULL) {
2844 		kmem_free(igb->mcast_table,
2845 		    igb->mcast_alloc_count * sizeof (struct ether_addr));
2846 		igb->mcast_table = NULL;
2847 	}
2848 }
2849 
2850 /*
2851  * igb_setup_multicast - setup multicast data structures
2852  *
2853  * This routine initializes all of the multicast related structures
2854  * and save them in the hardware registers.
2855  */
2856 static void
2857 igb_setup_multicst(igb_t *igb)
2858 {
2859 	uint8_t *mc_addr_list;
2860 	uint32_t mc_addr_count;
2861 	struct e1000_hw *hw = &igb->hw;
2862 
2863 	ASSERT(mutex_owned(&igb->gen_lock));
2864 	ASSERT(igb->mcast_count <= igb->mcast_max_num);
2865 
2866 	mc_addr_list = (uint8_t *)igb->mcast_table;
2867 	mc_addr_count = igb->mcast_count;
2868 
2869 	/*
2870 	 * Update the multicase addresses to the MTA registers
2871 	 */
2872 	e1000_update_mc_addr_list(hw, mc_addr_list, mc_addr_count);
2873 }
2874 
2875 /*
2876  * igb_get_conf - Get driver configurations set in driver.conf
2877  *
2878  * This routine gets user-configured values out of the configuration
2879  * file igb.conf.
2880  *
2881  * For each configurable value, there is a minimum, a maximum, and a
2882  * default.
2883  * If user does not configure a value, use the default.
2884  * If user configures below the minimum, use the minumum.
2885  * If user configures above the maximum, use the maxumum.
2886  */
2887 static void
2888 igb_get_conf(igb_t *igb)
2889 {
2890 	struct e1000_hw *hw = &igb->hw;
2891 	uint32_t default_mtu;
2892 	uint32_t flow_control;
2893 	uint32_t ring_per_group;
2894 	int i;
2895 
2896 	/*
2897 	 * igb driver supports the following user configurations:
2898 	 *
2899 	 * Link configurations:
2900 	 *    adv_autoneg_cap
2901 	 *    adv_1000fdx_cap
2902 	 *    adv_100fdx_cap
2903 	 *    adv_100hdx_cap
2904 	 *    adv_10fdx_cap
2905 	 *    adv_10hdx_cap
2906 	 * Note: 1000hdx is not supported.
2907 	 *
2908 	 * Jumbo frame configuration:
2909 	 *    default_mtu
2910 	 *
2911 	 * Ethernet flow control configuration:
2912 	 *    flow_control
2913 	 *
2914 	 * Multiple rings configurations:
2915 	 *    tx_queue_number
2916 	 *    tx_ring_size
2917 	 *    rx_queue_number
2918 	 *    rx_ring_size
2919 	 *
2920 	 * Call igb_get_prop() to get the value for a specific
2921 	 * configuration parameter.
2922 	 */
2923 
2924 	/*
2925 	 * Link configurations
2926 	 */
2927 	igb->param_adv_autoneg_cap = igb_get_prop(igb,
2928 	    PROP_ADV_AUTONEG_CAP, 0, 1, 1);
2929 	igb->param_adv_1000fdx_cap = igb_get_prop(igb,
2930 	    PROP_ADV_1000FDX_CAP, 0, 1, 1);
2931 	igb->param_adv_100fdx_cap = igb_get_prop(igb,
2932 	    PROP_ADV_100FDX_CAP, 0, 1, 1);
2933 	igb->param_adv_100hdx_cap = igb_get_prop(igb,
2934 	    PROP_ADV_100HDX_CAP, 0, 1, 1);
2935 	igb->param_adv_10fdx_cap = igb_get_prop(igb,
2936 	    PROP_ADV_10FDX_CAP, 0, 1, 1);
2937 	igb->param_adv_10hdx_cap = igb_get_prop(igb,
2938 	    PROP_ADV_10HDX_CAP, 0, 1, 1);
2939 
2940 	/*
2941 	 * Jumbo frame configurations
2942 	 */
2943 	default_mtu = igb_get_prop(igb, PROP_DEFAULT_MTU,
2944 	    MIN_MTU, MAX_MTU, DEFAULT_MTU);
2945 
2946 	igb->max_frame_size = default_mtu +
2947 	    sizeof (struct ether_vlan_header) + ETHERFCSL;
2948 
2949 	/*
2950 	 * Ethernet flow control configuration
2951 	 */
2952 	flow_control = igb_get_prop(igb, PROP_FLOW_CONTROL,
2953 	    e1000_fc_none, 4, e1000_fc_full);
2954 	if (flow_control == 4)
2955 		flow_control = e1000_fc_default;
2956 
2957 	hw->fc.requested_mode = flow_control;
2958 
2959 	/*
2960 	 * Multiple rings configurations
2961 	 */
2962 	igb->tx_ring_size = igb_get_prop(igb, PROP_TX_RING_SIZE,
2963 	    MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE);
2964 	igb->rx_ring_size = igb_get_prop(igb, PROP_RX_RING_SIZE,
2965 	    MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE);
2966 
2967 	igb->mr_enable = igb_get_prop(igb, PROP_MR_ENABLE, 0, 1, 0);
2968 	igb->num_rx_groups = igb_get_prop(igb, PROP_RX_GROUP_NUM,
2969 	    MIN_RX_GROUP_NUM, MAX_RX_GROUP_NUM, DEFAULT_RX_GROUP_NUM);
2970 	/*
2971 	 * Currently we do not support VMDq for 82576 and 82580.
2972 	 * If it is e1000_82576, set num_rx_groups to 1.
2973 	 */
2974 	if (hw->mac.type >= e1000_82576)
2975 		igb->num_rx_groups = 1;
2976 
2977 	if (igb->mr_enable) {
2978 		igb->num_tx_rings = igb->capab->def_tx_que_num;
2979 		igb->num_rx_rings = igb->capab->def_rx_que_num;
2980 	} else {
2981 		igb->num_tx_rings = 1;
2982 		igb->num_rx_rings = 1;
2983 
2984 		if (igb->num_rx_groups > 1) {
2985 			igb_log(igb, IGB_LOG_ERROR,
2986 			    "Invalid rx groups number. Please enable multiple "
2987 			    "rings first");
2988 			igb->num_rx_groups = 1;
2989 		}
2990 	}
2991 
2992 	/*
2993 	 * Check the divisibility between rx rings and rx groups.
2994 	 */
2995 	for (i = igb->num_rx_groups; i > 0; i--) {
2996 		if ((igb->num_rx_rings % i) == 0)
2997 			break;
2998 	}
2999 	if (i != igb->num_rx_groups) {
3000 		igb_log(igb, IGB_LOG_ERROR,
3001 		    "Invalid rx groups number. Downgrade the rx group "
3002 		    "number to %d.", i);
3003 		igb->num_rx_groups = i;
3004 	}
3005 
3006 	/*
3007 	 * Get the ring number per group.
3008 	 */
3009 	ring_per_group = igb->num_rx_rings / igb->num_rx_groups;
3010 
3011 	if (igb->num_rx_groups == 1) {
3012 		/*
3013 		 * One rx ring group, the rx ring number is num_rx_rings.
3014 		 */
3015 		igb->vmdq_mode = E1000_VMDQ_OFF;
3016 	} else if (ring_per_group == 1) {
3017 		/*
3018 		 * Multiple rx groups, each group has one rx ring.
3019 		 */
3020 		igb->vmdq_mode = E1000_VMDQ_MAC;
3021 	} else {
3022 		/*
3023 		 * Multiple groups and multiple rings.
3024 		 */
3025 		igb->vmdq_mode = E1000_VMDQ_MAC_RSS;
3026 	}
3027 
3028 	/*
3029 	 * Tunable used to force an interrupt type. The only use is
3030 	 * for testing of the lesser interrupt types.
3031 	 * 0 = don't force interrupt type
3032 	 * 1 = force interrupt type MSIX
3033 	 * 2 = force interrupt type MSI
3034 	 * 3 = force interrupt type Legacy
3035 	 */
3036 	igb->intr_force = igb_get_prop(igb, PROP_INTR_FORCE,
3037 	    IGB_INTR_NONE, IGB_INTR_LEGACY, IGB_INTR_NONE);
3038 
3039 	igb->tx_hcksum_enable = igb_get_prop(igb, PROP_TX_HCKSUM_ENABLE,
3040 	    0, 1, 1);
3041 	igb->rx_hcksum_enable = igb_get_prop(igb, PROP_RX_HCKSUM_ENABLE,
3042 	    0, 1, 1);
3043 	igb->lso_enable = igb_get_prop(igb, PROP_LSO_ENABLE,
3044 	    0, 1, 1);
3045 	igb->tx_head_wb_enable = igb_get_prop(igb, PROP_TX_HEAD_WB_ENABLE,
3046 	    0, 1, 1);
3047 
3048 	/*
3049 	 * igb LSO needs the tx h/w checksum support.
3050 	 * Here LSO will be disabled if tx h/w checksum has been disabled.
3051 	 */
3052 	if (igb->tx_hcksum_enable == B_FALSE)
3053 		igb->lso_enable = B_FALSE;
3054 
3055 	igb->tx_copy_thresh = igb_get_prop(igb, PROP_TX_COPY_THRESHOLD,
3056 	    MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
3057 	    DEFAULT_TX_COPY_THRESHOLD);
3058 	igb->tx_recycle_thresh = igb_get_prop(igb, PROP_TX_RECYCLE_THRESHOLD,
3059 	    MIN_TX_RECYCLE_THRESHOLD, MAX_TX_RECYCLE_THRESHOLD,
3060 	    DEFAULT_TX_RECYCLE_THRESHOLD);
3061 	igb->tx_overload_thresh = igb_get_prop(igb, PROP_TX_OVERLOAD_THRESHOLD,
3062 	    MIN_TX_OVERLOAD_THRESHOLD, MAX_TX_OVERLOAD_THRESHOLD,
3063 	    DEFAULT_TX_OVERLOAD_THRESHOLD);
3064 	igb->tx_resched_thresh = igb_get_prop(igb, PROP_TX_RESCHED_THRESHOLD,
3065 	    MIN_TX_RESCHED_THRESHOLD,
3066 	    MIN(igb->tx_ring_size, MAX_TX_RESCHED_THRESHOLD),
3067 	    igb->tx_ring_size > DEFAULT_TX_RESCHED_THRESHOLD ?
3068 	    DEFAULT_TX_RESCHED_THRESHOLD : DEFAULT_TX_RESCHED_THRESHOLD_LOW);
3069 
3070 	igb->rx_copy_thresh = igb_get_prop(igb, PROP_RX_COPY_THRESHOLD,
3071 	    MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
3072 	    DEFAULT_RX_COPY_THRESHOLD);
3073 	igb->rx_limit_per_intr = igb_get_prop(igb, PROP_RX_LIMIT_PER_INTR,
3074 	    MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
3075 	    DEFAULT_RX_LIMIT_PER_INTR);
3076 
3077 	igb->intr_throttling[0] = igb_get_prop(igb, PROP_INTR_THROTTLING,
3078 	    igb->capab->min_intr_throttle,
3079 	    igb->capab->max_intr_throttle,
3080 	    igb->capab->def_intr_throttle);
3081 
3082 	/*
3083 	 * Max number of multicast addresses
3084 	 */
3085 	igb->mcast_max_num =
3086 	    igb_get_prop(igb, PROP_MCAST_MAX_NUM,
3087 	    MIN_MCAST_NUM, MAX_MCAST_NUM, DEFAULT_MCAST_NUM);
3088 }
3089 
3090 /*
3091  * igb_get_prop - Get a property value out of the configuration file igb.conf
3092  *
3093  * Caller provides the name of the property, a default value, a minimum
3094  * value, and a maximum value.
3095  *
3096  * Return configured value of the property, with default, minimum and
3097  * maximum properly applied.
3098  */
3099 static int
3100 igb_get_prop(igb_t *igb,
3101     char *propname,	/* name of the property */
3102     int minval,		/* minimum acceptable value */
3103     int maxval,		/* maximim acceptable value */
3104     int defval)		/* default value */
3105 {
3106 	int value;
3107 
3108 	/*
3109 	 * Call ddi_prop_get_int() to read the conf settings
3110 	 */
3111 	value = ddi_prop_get_int(DDI_DEV_T_ANY, igb->dip,
3112 	    DDI_PROP_DONTPASS, propname, defval);
3113 
3114 	if (value > maxval)
3115 		value = maxval;
3116 
3117 	if (value < minval)
3118 		value = minval;
3119 
3120 	return (value);
3121 }
3122 
3123 /*
3124  * igb_setup_link - Using the link properties to setup the link
3125  */
3126 int
3127 igb_setup_link(igb_t *igb, boolean_t setup_hw)
3128 {
3129 	struct e1000_mac_info *mac;
3130 	struct e1000_phy_info *phy;
3131 	boolean_t invalid;
3132 
3133 	mac = &igb->hw.mac;
3134 	phy = &igb->hw.phy;
3135 	invalid = B_FALSE;
3136 
3137 	if (igb->param_adv_autoneg_cap == 1) {
3138 		mac->autoneg = B_TRUE;
3139 		phy->autoneg_advertised = 0;
3140 
3141 		/*
3142 		 * 1000hdx is not supported for autonegotiation
3143 		 */
3144 		if (igb->param_adv_1000fdx_cap == 1)
3145 			phy->autoneg_advertised |= ADVERTISE_1000_FULL;
3146 
3147 		if (igb->param_adv_100fdx_cap == 1)
3148 			phy->autoneg_advertised |= ADVERTISE_100_FULL;
3149 
3150 		if (igb->param_adv_100hdx_cap == 1)
3151 			phy->autoneg_advertised |= ADVERTISE_100_HALF;
3152 
3153 		if (igb->param_adv_10fdx_cap == 1)
3154 			phy->autoneg_advertised |= ADVERTISE_10_FULL;
3155 
3156 		if (igb->param_adv_10hdx_cap == 1)
3157 			phy->autoneg_advertised |= ADVERTISE_10_HALF;
3158 
3159 		if (phy->autoneg_advertised == 0)
3160 			invalid = B_TRUE;
3161 	} else {
3162 		mac->autoneg = B_FALSE;
3163 
3164 		/*
3165 		 * 1000fdx and 1000hdx are not supported for forced link
3166 		 */
3167 		if (igb->param_adv_100fdx_cap == 1)
3168 			mac->forced_speed_duplex = ADVERTISE_100_FULL;
3169 		else if (igb->param_adv_100hdx_cap == 1)
3170 			mac->forced_speed_duplex = ADVERTISE_100_HALF;
3171 		else if (igb->param_adv_10fdx_cap == 1)
3172 			mac->forced_speed_duplex = ADVERTISE_10_FULL;
3173 		else if (igb->param_adv_10hdx_cap == 1)
3174 			mac->forced_speed_duplex = ADVERTISE_10_HALF;
3175 		else
3176 			invalid = B_TRUE;
3177 	}
3178 
3179 	if (invalid) {
3180 		igb_log(igb, IGB_LOG_INFO, "Invalid link settings. Setup "
3181 		    "link to autonegotiation with full link capabilities.");
3182 		mac->autoneg = B_TRUE;
3183 		phy->autoneg_advertised = ADVERTISE_1000_FULL |
3184 		    ADVERTISE_100_FULL | ADVERTISE_100_HALF |
3185 		    ADVERTISE_10_FULL | ADVERTISE_10_HALF;
3186 	}
3187 
3188 	if (setup_hw) {
3189 		if (e1000_setup_link(&igb->hw) != E1000_SUCCESS)
3190 			return (IGB_FAILURE);
3191 	}
3192 
3193 	return (IGB_SUCCESS);
3194 }
3195 
3196 
3197 /*
3198  * igb_is_link_up - Check if the link is up
3199  */
3200 static boolean_t
3201 igb_is_link_up(igb_t *igb)
3202 {
3203 	struct e1000_hw *hw = &igb->hw;
3204 	boolean_t link_up = B_FALSE;
3205 
3206 	ASSERT(mutex_owned(&igb->gen_lock));
3207 
3208 	/*
3209 	 * get_link_status is set in the interrupt handler on link-status-change
3210 	 * or rx sequence error interrupt.  get_link_status will stay
3211 	 * false until the e1000_check_for_link establishes link only
3212 	 * for copper adapters.
3213 	 */
3214 	switch (hw->phy.media_type) {
3215 	case e1000_media_type_copper:
3216 		if (hw->mac.get_link_status) {
3217 			(void) e1000_check_for_link(hw);
3218 			link_up = !hw->mac.get_link_status;
3219 		} else {
3220 			link_up = B_TRUE;
3221 		}
3222 		break;
3223 	case e1000_media_type_fiber:
3224 		(void) e1000_check_for_link(hw);
3225 		link_up = (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU);
3226 		break;
3227 	case e1000_media_type_internal_serdes:
3228 		(void) e1000_check_for_link(hw);
3229 		link_up = hw->mac.serdes_has_link;
3230 		break;
3231 	}
3232 
3233 	return (link_up);
3234 }
3235 
3236 /*
3237  * igb_link_check - Link status processing
3238  */
3239 static boolean_t
3240 igb_link_check(igb_t *igb)
3241 {
3242 	struct e1000_hw *hw = &igb->hw;
3243 	uint16_t speed = 0, duplex = 0;
3244 	boolean_t link_changed = B_FALSE;
3245 
3246 	ASSERT(mutex_owned(&igb->gen_lock));
3247 
3248 	if (igb_is_link_up(igb)) {
3249 		/*
3250 		 * The Link is up, check whether it was marked as down earlier
3251 		 */
3252 		if (igb->link_state != LINK_STATE_UP) {
3253 			(void) e1000_get_speed_and_duplex(hw, &speed, &duplex);
3254 			igb->link_speed = speed;
3255 			igb->link_duplex = duplex;
3256 			igb->link_state = LINK_STATE_UP;
3257 			link_changed = B_TRUE;
3258 			if (!igb->link_complete)
3259 				igb_stop_link_timer(igb);
3260 		}
3261 	} else if (igb->link_complete) {
3262 		if (igb->link_state != LINK_STATE_DOWN) {
3263 			igb->link_speed = 0;
3264 			igb->link_duplex = 0;
3265 			igb->link_state = LINK_STATE_DOWN;
3266 			link_changed = B_TRUE;
3267 		}
3268 	}
3269 
3270 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
3271 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
3272 		return (B_FALSE);
3273 	}
3274 
3275 	return (link_changed);
3276 }
3277 
3278 /*
3279  * igb_local_timer - driver watchdog function
3280  *
3281  * This function will handle the hardware stall check, link status
3282  * check and other routines.
3283  */
3284 static void
3285 igb_local_timer(void *arg)
3286 {
3287 	igb_t *igb = (igb_t *)arg;
3288 	boolean_t link_changed = B_FALSE;
3289 
3290 	if (igb->igb_state & IGB_ERROR) {
3291 		igb->reset_count++;
3292 		if (igb_reset(igb) == IGB_SUCCESS)
3293 			ddi_fm_service_impact(igb->dip, DDI_SERVICE_RESTORED);
3294 
3295 		igb_restart_watchdog_timer(igb);
3296 		return;
3297 	}
3298 
3299 	if (igb_stall_check(igb) || (igb->igb_state & IGB_STALL)) {
3300 		igb_fm_ereport(igb, DDI_FM_DEVICE_STALL);
3301 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
3302 		igb->reset_count++;
3303 		if (igb_reset(igb) == IGB_SUCCESS)
3304 			ddi_fm_service_impact(igb->dip, DDI_SERVICE_RESTORED);
3305 
3306 		igb_restart_watchdog_timer(igb);
3307 		return;
3308 	}
3309 
3310 	mutex_enter(&igb->gen_lock);
3311 	if (!(igb->igb_state & IGB_SUSPENDED) && (igb->igb_state & IGB_STARTED))
3312 		link_changed = igb_link_check(igb);
3313 	mutex_exit(&igb->gen_lock);
3314 
3315 	if (link_changed)
3316 		mac_link_update(igb->mac_hdl, igb->link_state);
3317 
3318 	igb_restart_watchdog_timer(igb);
3319 }
3320 
3321 /*
3322  * igb_link_timer - link setup timer function
3323  *
3324  * It is called when the timer for link setup is expired, which indicates
3325  * the completion of the link setup. The link state will not be updated
3326  * until the link setup is completed. And the link state will not be sent
3327  * to the upper layer through mac_link_update() in this function. It will
3328  * be updated in the local timer routine or the interrupts service routine
3329  * after the interface is started (plumbed).
3330  */
3331 static void
3332 igb_link_timer(void *arg)
3333 {
3334 	igb_t *igb = (igb_t *)arg;
3335 
3336 	mutex_enter(&igb->link_lock);
3337 	igb->link_complete = B_TRUE;
3338 	igb->link_tid = 0;
3339 	mutex_exit(&igb->link_lock);
3340 }
3341 /*
3342  * igb_stall_check - check for transmit stall
3343  *
3344  * This function checks if the adapter is stalled (in transmit).
3345  *
3346  * It is called each time the watchdog timeout is invoked.
3347  * If the transmit descriptor reclaim continuously fails,
3348  * the watchdog value will increment by 1. If the watchdog
3349  * value exceeds the threshold, the igb is assumed to
3350  * have stalled and need to be reset.
3351  */
3352 static boolean_t
3353 igb_stall_check(igb_t *igb)
3354 {
3355 	igb_tx_ring_t *tx_ring;
3356 	struct e1000_hw *hw = &igb->hw;
3357 	boolean_t result;
3358 	int i;
3359 
3360 	if (igb->link_state != LINK_STATE_UP)
3361 		return (B_FALSE);
3362 
3363 	/*
3364 	 * If any tx ring is stalled, we'll reset the chipset
3365 	 */
3366 	result = B_FALSE;
3367 	for (i = 0; i < igb->num_tx_rings; i++) {
3368 		tx_ring = &igb->tx_rings[i];
3369 
3370 		if (tx_ring->recycle_fail > 0)
3371 			tx_ring->stall_watchdog++;
3372 		else
3373 			tx_ring->stall_watchdog = 0;
3374 
3375 		if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) {
3376 			result = B_TRUE;
3377 			if (hw->mac.type == e1000_82580) {
3378 				hw->dev_spec._82575.global_device_reset
3379 				    = B_TRUE;
3380 			}
3381 			break;
3382 		}
3383 	}
3384 
3385 	if (result) {
3386 		tx_ring->stall_watchdog = 0;
3387 		tx_ring->recycle_fail = 0;
3388 	}
3389 
3390 	return (result);
3391 }
3392 
3393 
3394 /*
3395  * is_valid_mac_addr - Check if the mac address is valid
3396  */
3397 static boolean_t
3398 is_valid_mac_addr(uint8_t *mac_addr)
3399 {
3400 	const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 };
3401 	const uint8_t addr_test2[6] =
3402 	    { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
3403 
3404 	if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) ||
3405 	    !(bcmp(addr_test2, mac_addr, ETHERADDRL)))
3406 		return (B_FALSE);
3407 
3408 	return (B_TRUE);
3409 }
3410 
3411 static boolean_t
3412 igb_find_mac_address(igb_t *igb)
3413 {
3414 	struct e1000_hw *hw = &igb->hw;
3415 #ifdef __sparc
3416 	uchar_t *bytes;
3417 	struct ether_addr sysaddr;
3418 	uint_t nelts;
3419 	int err;
3420 	boolean_t found = B_FALSE;
3421 
3422 	/*
3423 	 * The "vendor's factory-set address" may already have
3424 	 * been extracted from the chip, but if the property
3425 	 * "local-mac-address" is set we use that instead.
3426 	 *
3427 	 * We check whether it looks like an array of 6
3428 	 * bytes (which it should, if OBP set it).  If we can't
3429 	 * make sense of it this way, we'll ignore it.
3430 	 */
3431 	err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip,
3432 	    DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts);
3433 	if (err == DDI_PROP_SUCCESS) {
3434 		if (nelts == ETHERADDRL) {
3435 			while (nelts--)
3436 				hw->mac.addr[nelts] = bytes[nelts];
3437 			found = B_TRUE;
3438 		}
3439 		ddi_prop_free(bytes);
3440 	}
3441 
3442 	/*
3443 	 * Look up the OBP property "local-mac-address?". If the user has set
3444 	 * 'local-mac-address? = false', use "the system address" instead.
3445 	 */
3446 	if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip, 0,
3447 	    "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
3448 		if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
3449 			if (localetheraddr(NULL, &sysaddr) != 0) {
3450 				bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
3451 				found = B_TRUE;
3452 			}
3453 		}
3454 		ddi_prop_free(bytes);
3455 	}
3456 
3457 	/*
3458 	 * Finally(!), if there's a valid "mac-address" property (created
3459 	 * if we netbooted from this interface), we must use this instead
3460 	 * of any of the above to ensure that the NFS/install server doesn't
3461 	 * get confused by the address changing as Solaris takes over!
3462 	 */
3463 	err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip,
3464 	    DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
3465 	if (err == DDI_PROP_SUCCESS) {
3466 		if (nelts == ETHERADDRL) {
3467 			while (nelts--)
3468 				hw->mac.addr[nelts] = bytes[nelts];
3469 			found = B_TRUE;
3470 		}
3471 		ddi_prop_free(bytes);
3472 	}
3473 
3474 	if (found) {
3475 		bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
3476 		return (B_TRUE);
3477 	}
3478 #endif
3479 
3480 	/*
3481 	 * Read the device MAC address from the EEPROM
3482 	 */
3483 	if (e1000_read_mac_addr(hw) != E1000_SUCCESS)
3484 		return (B_FALSE);
3485 
3486 	return (B_TRUE);
3487 }
3488 
3489 #pragma inline(igb_arm_watchdog_timer)
3490 
3491 static void
3492 igb_arm_watchdog_timer(igb_t *igb)
3493 {
3494 	/*
3495 	 * Fire a watchdog timer
3496 	 */
3497 	igb->watchdog_tid =
3498 	    timeout(igb_local_timer,
3499 	    (void *)igb, 1 * drv_usectohz(1000000));
3500 
3501 }
3502 
3503 /*
3504  * igb_enable_watchdog_timer - Enable and start the driver watchdog timer
3505  */
3506 void
3507 igb_enable_watchdog_timer(igb_t *igb)
3508 {
3509 	mutex_enter(&igb->watchdog_lock);
3510 
3511 	if (!igb->watchdog_enable) {
3512 		igb->watchdog_enable = B_TRUE;
3513 		igb->watchdog_start = B_TRUE;
3514 		igb_arm_watchdog_timer(igb);
3515 	}
3516 
3517 	mutex_exit(&igb->watchdog_lock);
3518 
3519 }
3520 
3521 /*
3522  * igb_disable_watchdog_timer - Disable and stop the driver watchdog timer
3523  */
3524 void
3525 igb_disable_watchdog_timer(igb_t *igb)
3526 {
3527 	timeout_id_t tid;
3528 
3529 	mutex_enter(&igb->watchdog_lock);
3530 
3531 	igb->watchdog_enable = B_FALSE;
3532 	igb->watchdog_start = B_FALSE;
3533 	tid = igb->watchdog_tid;
3534 	igb->watchdog_tid = 0;
3535 
3536 	mutex_exit(&igb->watchdog_lock);
3537 
3538 	if (tid != 0)
3539 		(void) untimeout(tid);
3540 
3541 }
3542 
3543 /*
3544  * igb_start_watchdog_timer - Start the driver watchdog timer
3545  */
3546 static void
3547 igb_start_watchdog_timer(igb_t *igb)
3548 {
3549 	mutex_enter(&igb->watchdog_lock);
3550 
3551 	if (igb->watchdog_enable) {
3552 		if (!igb->watchdog_start) {
3553 			igb->watchdog_start = B_TRUE;
3554 			igb_arm_watchdog_timer(igb);
3555 		}
3556 	}
3557 
3558 	mutex_exit(&igb->watchdog_lock);
3559 }
3560 
3561 /*
3562  * igb_restart_watchdog_timer - Restart the driver watchdog timer
3563  */
3564 static void
3565 igb_restart_watchdog_timer(igb_t *igb)
3566 {
3567 	mutex_enter(&igb->watchdog_lock);
3568 
3569 	if (igb->watchdog_start)
3570 		igb_arm_watchdog_timer(igb);
3571 
3572 	mutex_exit(&igb->watchdog_lock);
3573 }
3574 
3575 /*
3576  * igb_stop_watchdog_timer - Stop the driver watchdog timer
3577  */
3578 static void
3579 igb_stop_watchdog_timer(igb_t *igb)
3580 {
3581 	timeout_id_t tid;
3582 
3583 	mutex_enter(&igb->watchdog_lock);
3584 
3585 	igb->watchdog_start = B_FALSE;
3586 	tid = igb->watchdog_tid;
3587 	igb->watchdog_tid = 0;
3588 
3589 	mutex_exit(&igb->watchdog_lock);
3590 
3591 	if (tid != 0)
3592 		(void) untimeout(tid);
3593 }
3594 
3595 /*
3596  * igb_start_link_timer - Start the link setup timer
3597  */
3598 static void
3599 igb_start_link_timer(struct igb *igb)
3600 {
3601 	struct e1000_hw *hw = &igb->hw;
3602 	clock_t link_timeout;
3603 
3604 	if (hw->mac.autoneg)
3605 		link_timeout = PHY_AUTO_NEG_LIMIT *
3606 		    drv_usectohz(100000);
3607 	else
3608 		link_timeout = PHY_FORCE_LIMIT * drv_usectohz(100000);
3609 
3610 	mutex_enter(&igb->link_lock);
3611 	if (hw->phy.autoneg_wait_to_complete) {
3612 		igb->link_complete = B_TRUE;
3613 	} else {
3614 		igb->link_complete = B_FALSE;
3615 		igb->link_tid = timeout(igb_link_timer, (void *)igb,
3616 		    link_timeout);
3617 	}
3618 	mutex_exit(&igb->link_lock);
3619 }
3620 
3621 /*
3622  * igb_stop_link_timer - Stop the link setup timer
3623  */
3624 static void
3625 igb_stop_link_timer(struct igb *igb)
3626 {
3627 	timeout_id_t tid;
3628 
3629 	mutex_enter(&igb->link_lock);
3630 	igb->link_complete = B_TRUE;
3631 	tid = igb->link_tid;
3632 	igb->link_tid = 0;
3633 	mutex_exit(&igb->link_lock);
3634 
3635 	if (tid != 0)
3636 		(void) untimeout(tid);
3637 }
3638 
3639 /*
3640  * igb_disable_adapter_interrupts - Clear/disable all hardware interrupts
3641  */
3642 static void
3643 igb_disable_adapter_interrupts(igb_t *igb)
3644 {
3645 	struct e1000_hw *hw = &igb->hw;
3646 
3647 	/*
3648 	 * Set the IMC register to mask all the interrupts,
3649 	 * including the tx interrupts.
3650 	 */
3651 	E1000_WRITE_REG(hw, E1000_IMC, ~0);
3652 	E1000_WRITE_REG(hw, E1000_IAM, 0);
3653 
3654 	/*
3655 	 * Additional disabling for MSI-X
3656 	 */
3657 	if (igb->intr_type == DDI_INTR_TYPE_MSIX) {
3658 		E1000_WRITE_REG(hw, E1000_EIMC, ~0);
3659 		E1000_WRITE_REG(hw, E1000_EIAC, 0);
3660 		E1000_WRITE_REG(hw, E1000_EIAM, 0);
3661 	}
3662 
3663 	E1000_WRITE_FLUSH(hw);
3664 }
3665 
3666 /*
3667  * igb_enable_adapter_interrupts_82580 - Enable NIC interrupts for 82580
3668  */
3669 static void
3670 igb_enable_adapter_interrupts_82580(igb_t *igb)
3671 {
3672 	struct e1000_hw *hw = &igb->hw;
3673 
3674 	/* Clear any pending interrupts */
3675 	(void) E1000_READ_REG(hw, E1000_ICR);
3676 	igb->ims_mask |= E1000_IMS_DRSTA;
3677 
3678 	if (igb->intr_type == DDI_INTR_TYPE_MSIX) {
3679 
3680 		/* Interrupt enabling for MSI-X */
3681 		E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask);
3682 		E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask);
3683 		igb->ims_mask = (E1000_IMS_LSC | E1000_IMS_DRSTA);
3684 		E1000_WRITE_REG(hw, E1000_IMS, igb->ims_mask);
3685 	} else { /* Interrupt enabling for MSI and legacy */
3686 		E1000_WRITE_REG(hw, E1000_IVAR0, E1000_IVAR_VALID);
3687 		igb->ims_mask = IMS_ENABLE_MASK | E1000_IMS_TXQE;
3688 		igb->ims_mask |= E1000_IMS_DRSTA;
3689 		E1000_WRITE_REG(hw, E1000_IMS, igb->ims_mask);
3690 	}
3691 
3692 	/* Disable auto-mask for ICR interrupt bits */
3693 	E1000_WRITE_REG(hw, E1000_IAM, 0);
3694 
3695 	E1000_WRITE_FLUSH(hw);
3696 }
3697 
3698 /*
3699  * igb_enable_adapter_interrupts_82576 - Enable NIC interrupts for 82576
3700  */
3701 static void
3702 igb_enable_adapter_interrupts_82576(igb_t *igb)
3703 {
3704 	struct e1000_hw *hw = &igb->hw;
3705 
3706 	/* Clear any pending interrupts */
3707 	(void) E1000_READ_REG(hw, E1000_ICR);
3708 
3709 	if (igb->intr_type == DDI_INTR_TYPE_MSIX) {
3710 
3711 		/* Interrupt enabling for MSI-X */
3712 		E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask);
3713 		E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask);
3714 		igb->ims_mask = E1000_IMS_LSC;
3715 		E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC);
3716 	} else {
3717 		/* Interrupt enabling for MSI and legacy */
3718 		E1000_WRITE_REG(hw, E1000_IVAR0, E1000_IVAR_VALID);
3719 		igb->ims_mask = IMS_ENABLE_MASK | E1000_IMS_TXQE;
3720 		E1000_WRITE_REG(hw, E1000_IMS,
3721 		    (IMS_ENABLE_MASK | E1000_IMS_TXQE));
3722 	}
3723 
3724 	/* Disable auto-mask for ICR interrupt bits */
3725 	E1000_WRITE_REG(hw, E1000_IAM, 0);
3726 
3727 	E1000_WRITE_FLUSH(hw);
3728 }
3729 
3730 /*
3731  * igb_enable_adapter_interrupts_82575 - Enable NIC interrupts for 82575
3732  */
3733 static void
3734 igb_enable_adapter_interrupts_82575(igb_t *igb)
3735 {
3736 	struct e1000_hw *hw = &igb->hw;
3737 	uint32_t reg;
3738 
3739 	/* Clear any pending interrupts */
3740 	(void) E1000_READ_REG(hw, E1000_ICR);
3741 
3742 	if (igb->intr_type == DDI_INTR_TYPE_MSIX) {
3743 		/* Interrupt enabling for MSI-X */
3744 		E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask);
3745 		E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask);
3746 		igb->ims_mask = E1000_IMS_LSC;
3747 		E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC);
3748 
3749 		/* Enable MSI-X PBA support */
3750 		reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
3751 		reg |= E1000_CTRL_EXT_PBA_CLR;
3752 
3753 		/* Non-selective interrupt clear-on-read */
3754 		reg |= E1000_CTRL_EXT_IRCA;	/* Called NSICR in the EAS */
3755 
3756 		E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
3757 	} else {
3758 		/* Interrupt enabling for MSI and legacy */
3759 		igb->ims_mask = IMS_ENABLE_MASK;
3760 		E1000_WRITE_REG(hw, E1000_IMS, IMS_ENABLE_MASK);
3761 	}
3762 
3763 	E1000_WRITE_FLUSH(hw);
3764 }
3765 
3766 /*
3767  * Loopback Support
3768  */
3769 static lb_property_t lb_normal =
3770 	{ normal,	"normal",	IGB_LB_NONE		};
3771 static lb_property_t lb_external =
3772 	{ external,	"External",	IGB_LB_EXTERNAL		};
3773 static lb_property_t lb_phy =
3774 	{ internal,	"PHY",		IGB_LB_INTERNAL_PHY	};
3775 static lb_property_t lb_serdes =
3776 	{ internal,	"SerDes",	IGB_LB_INTERNAL_SERDES	};
3777 
3778 enum ioc_reply
3779 igb_loopback_ioctl(igb_t *igb, struct iocblk *iocp, mblk_t *mp)
3780 {
3781 	lb_info_sz_t *lbsp;
3782 	lb_property_t *lbpp;
3783 	struct e1000_hw *hw;
3784 	uint32_t *lbmp;
3785 	uint32_t size;
3786 	uint32_t value;
3787 
3788 	hw = &igb->hw;
3789 
3790 	if (mp->b_cont == NULL)
3791 		return (IOC_INVAL);
3792 
3793 	switch (iocp->ioc_cmd) {
3794 	default:
3795 		return (IOC_INVAL);
3796 
3797 	case LB_GET_INFO_SIZE:
3798 		size = sizeof (lb_info_sz_t);
3799 		if (iocp->ioc_count != size)
3800 			return (IOC_INVAL);
3801 
3802 		value = sizeof (lb_normal);
3803 		if (hw->phy.media_type == e1000_media_type_copper)
3804 			value += sizeof (lb_phy);
3805 		else
3806 			value += sizeof (lb_serdes);
3807 		value += sizeof (lb_external);
3808 
3809 		lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr;
3810 		*lbsp = value;
3811 		break;
3812 
3813 	case LB_GET_INFO:
3814 		value = sizeof (lb_normal);
3815 		if (hw->phy.media_type == e1000_media_type_copper)
3816 			value += sizeof (lb_phy);
3817 		else
3818 			value += sizeof (lb_serdes);
3819 		value += sizeof (lb_external);
3820 
3821 		size = value;
3822 		if (iocp->ioc_count != size)
3823 			return (IOC_INVAL);
3824 
3825 		value = 0;
3826 		lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr;
3827 
3828 		lbpp[value++] = lb_normal;
3829 		if (hw->phy.media_type == e1000_media_type_copper)
3830 			lbpp[value++] = lb_phy;
3831 		else
3832 			lbpp[value++] = lb_serdes;
3833 		lbpp[value++] = lb_external;
3834 		break;
3835 
3836 	case LB_GET_MODE:
3837 		size = sizeof (uint32_t);
3838 		if (iocp->ioc_count != size)
3839 			return (IOC_INVAL);
3840 
3841 		lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3842 		*lbmp = igb->loopback_mode;
3843 		break;
3844 
3845 	case LB_SET_MODE:
3846 		size = 0;
3847 		if (iocp->ioc_count != sizeof (uint32_t))
3848 			return (IOC_INVAL);
3849 
3850 		lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3851 		if (!igb_set_loopback_mode(igb, *lbmp))
3852 			return (IOC_INVAL);
3853 		break;
3854 	}
3855 
3856 	iocp->ioc_count = size;
3857 	iocp->ioc_error = 0;
3858 
3859 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
3860 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
3861 		return (IOC_INVAL);
3862 	}
3863 
3864 	return (IOC_REPLY);
3865 }
3866 
3867 /*
3868  * igb_set_loopback_mode - Setup loopback based on the loopback mode
3869  */
3870 static boolean_t
3871 igb_set_loopback_mode(igb_t *igb, uint32_t mode)
3872 {
3873 	struct e1000_hw *hw;
3874 	int i;
3875 
3876 	if (mode == igb->loopback_mode)
3877 		return (B_TRUE);
3878 
3879 	hw = &igb->hw;
3880 
3881 	igb->loopback_mode = mode;
3882 
3883 	if (mode == IGB_LB_NONE) {
3884 		/* Reset the chip */
3885 		hw->phy.autoneg_wait_to_complete = B_TRUE;
3886 		(void) igb_reset(igb);
3887 		hw->phy.autoneg_wait_to_complete = B_FALSE;
3888 		return (B_TRUE);
3889 	}
3890 
3891 	mutex_enter(&igb->gen_lock);
3892 
3893 	switch (mode) {
3894 	default:
3895 		mutex_exit(&igb->gen_lock);
3896 		return (B_FALSE);
3897 
3898 	case IGB_LB_EXTERNAL:
3899 		igb_set_external_loopback(igb);
3900 		break;
3901 
3902 	case IGB_LB_INTERNAL_PHY:
3903 		igb_set_internal_phy_loopback(igb);
3904 		break;
3905 
3906 	case IGB_LB_INTERNAL_SERDES:
3907 		igb_set_internal_serdes_loopback(igb);
3908 		break;
3909 	}
3910 
3911 	mutex_exit(&igb->gen_lock);
3912 
3913 	/*
3914 	 * When external loopback is set, wait up to 1000ms to get the link up.
3915 	 * According to test, 1000ms can work and it's an experimental value.
3916 	 */
3917 	if (mode == IGB_LB_EXTERNAL) {
3918 		for (i = 0; i <= 10; i++) {
3919 			mutex_enter(&igb->gen_lock);
3920 			(void) igb_link_check(igb);
3921 			mutex_exit(&igb->gen_lock);
3922 
3923 			if (igb->link_state == LINK_STATE_UP)
3924 				break;
3925 
3926 			msec_delay(100);
3927 		}
3928 
3929 		if (igb->link_state != LINK_STATE_UP) {
3930 			/*
3931 			 * Does not support external loopback.
3932 			 * Reset driver to loopback none.
3933 			 */
3934 			igb->loopback_mode = IGB_LB_NONE;
3935 
3936 			/* Reset the chip */
3937 			hw->phy.autoneg_wait_to_complete = B_TRUE;
3938 			(void) igb_reset(igb);
3939 			hw->phy.autoneg_wait_to_complete = B_FALSE;
3940 
3941 			igb_log(igb, IGB_LOG_INFO, "Set external loopback "
3942 			    "failed, reset to loopback none.");
3943 
3944 			return (B_FALSE);
3945 		}
3946 	}
3947 
3948 	return (B_TRUE);
3949 }
3950 
3951 /*
3952  * igb_set_external_loopback - Set the external loopback mode
3953  */
3954 static void
3955 igb_set_external_loopback(igb_t *igb)
3956 {
3957 	struct e1000_hw *hw;
3958 	uint32_t ctrl_ext;
3959 
3960 	hw = &igb->hw;
3961 
3962 	/* Set link mode to PHY (00b) in the Extended Control register */
3963 	ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
3964 	ctrl_ext &= ~E1000_CTRL_EXT_LINK_MODE_MASK;
3965 	E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
3966 
3967 	(void) e1000_write_phy_reg(hw, 0x0, 0x0140);
3968 	(void) e1000_write_phy_reg(hw, 0x9, 0x1a00);
3969 	(void) e1000_write_phy_reg(hw, 0x12, 0x1610);
3970 	(void) e1000_write_phy_reg(hw, 0x1f37, 0x3f1c);
3971 }
3972 
3973 /*
3974  * igb_set_internal_phy_loopback - Set the internal PHY loopback mode
3975  */
3976 static void
3977 igb_set_internal_phy_loopback(igb_t *igb)
3978 {
3979 	struct e1000_hw *hw;
3980 	uint32_t ctrl_ext;
3981 	uint16_t phy_ctrl;
3982 	uint16_t phy_pconf;
3983 
3984 	hw = &igb->hw;
3985 
3986 	/* Set link mode to PHY (00b) in the Extended Control register */
3987 	ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
3988 	ctrl_ext &= ~E1000_CTRL_EXT_LINK_MODE_MASK;
3989 	E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
3990 
3991 	/*
3992 	 * Set PHY control register (0x4140):
3993 	 *    Set full duplex mode
3994 	 *    Set loopback bit
3995 	 *    Clear auto-neg enable bit
3996 	 *    Set PHY speed
3997 	 */
3998 	phy_ctrl = MII_CR_FULL_DUPLEX | MII_CR_SPEED_1000 | MII_CR_LOOPBACK;
3999 	(void) e1000_write_phy_reg(hw, PHY_CONTROL, phy_ctrl);
4000 
4001 	/* Set the link disable bit in the Port Configuration register */
4002 	(void) e1000_read_phy_reg(hw, 0x10, &phy_pconf);
4003 	phy_pconf |= (uint16_t)1 << 14;
4004 	(void) e1000_write_phy_reg(hw, 0x10, phy_pconf);
4005 }
4006 
4007 /*
4008  * igb_set_internal_serdes_loopback - Set the internal SerDes loopback mode
4009  */
4010 static void
4011 igb_set_internal_serdes_loopback(igb_t *igb)
4012 {
4013 	struct e1000_hw *hw;
4014 	uint32_t ctrl_ext;
4015 	uint32_t ctrl;
4016 	uint32_t pcs_lctl;
4017 	uint32_t connsw;
4018 
4019 	hw = &igb->hw;
4020 
4021 	/* Set link mode to SerDes (11b) in the Extended Control register */
4022 	ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
4023 	ctrl_ext |= E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
4024 	E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
4025 
4026 	/* Configure the SerDes to loopback */
4027 	E1000_WRITE_REG(hw, E1000_SCTL, 0x410);
4028 
4029 	/* Set Device Control register */
4030 	ctrl = E1000_READ_REG(hw, E1000_CTRL);
4031 	ctrl |= (E1000_CTRL_FD |	/* Force full duplex */
4032 	    E1000_CTRL_SLU);		/* Force link up */
4033 	ctrl &= ~(E1000_CTRL_RFCE |	/* Disable receive flow control */
4034 	    E1000_CTRL_TFCE |		/* Disable transmit flow control */
4035 	    E1000_CTRL_LRST);		/* Clear link reset */
4036 	E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
4037 
4038 	/* Set PCS Link Control register */
4039 	pcs_lctl = E1000_READ_REG(hw, E1000_PCS_LCTL);
4040 	pcs_lctl |= (E1000_PCS_LCTL_FORCE_LINK |
4041 	    E1000_PCS_LCTL_FSD |
4042 	    E1000_PCS_LCTL_FDV_FULL |
4043 	    E1000_PCS_LCTL_FLV_LINK_UP);
4044 	pcs_lctl &= ~E1000_PCS_LCTL_AN_ENABLE;
4045 	E1000_WRITE_REG(hw, E1000_PCS_LCTL, pcs_lctl);
4046 
4047 	/* Set the Copper/Fiber Switch Control - CONNSW register */
4048 	connsw = E1000_READ_REG(hw, E1000_CONNSW);
4049 	connsw &= ~E1000_CONNSW_ENRGSRC;
4050 	E1000_WRITE_REG(hw, E1000_CONNSW, connsw);
4051 }
4052 
4053 #pragma inline(igb_intr_rx_work)
4054 /*
4055  * igb_intr_rx_work - rx processing of ISR
4056  */
4057 static void
4058 igb_intr_rx_work(igb_rx_ring_t *rx_ring)
4059 {
4060 	mblk_t *mp;
4061 
4062 	mutex_enter(&rx_ring->rx_lock);
4063 	mp = igb_rx(rx_ring, IGB_NO_POLL);
4064 	mutex_exit(&rx_ring->rx_lock);
4065 
4066 	if (mp != NULL)
4067 		mac_rx_ring(rx_ring->igb->mac_hdl, rx_ring->ring_handle, mp,
4068 		    rx_ring->ring_gen_num);
4069 }
4070 
4071 #pragma inline(igb_intr_tx_work)
4072 /*
4073  * igb_intr_tx_work - tx processing of ISR
4074  */
4075 static void
4076 igb_intr_tx_work(igb_tx_ring_t *tx_ring)
4077 {
4078 	igb_t *igb = tx_ring->igb;
4079 
4080 	/* Recycle the tx descriptors */
4081 	tx_ring->tx_recycle(tx_ring);
4082 
4083 	/* Schedule the re-transmit */
4084 	if (tx_ring->reschedule &&
4085 	    (tx_ring->tbd_free >= igb->tx_resched_thresh)) {
4086 		tx_ring->reschedule = B_FALSE;
4087 		mac_tx_ring_update(tx_ring->igb->mac_hdl, tx_ring->ring_handle);
4088 		IGB_DEBUG_STAT(tx_ring->stat_reschedule);
4089 	}
4090 }
4091 
4092 #pragma inline(igb_intr_link_work)
4093 /*
4094  * igb_intr_link_work - link-status-change processing of ISR
4095  */
4096 static void
4097 igb_intr_link_work(igb_t *igb)
4098 {
4099 	boolean_t link_changed;
4100 
4101 	igb_stop_watchdog_timer(igb);
4102 
4103 	mutex_enter(&igb->gen_lock);
4104 
4105 	/*
4106 	 * Because we got a link-status-change interrupt, force
4107 	 * e1000_check_for_link() to look at phy
4108 	 */
4109 	igb->hw.mac.get_link_status = B_TRUE;
4110 
4111 	/* igb_link_check takes care of link status change */
4112 	link_changed = igb_link_check(igb);
4113 
4114 	/* Get new phy state */
4115 	igb_get_phy_state(igb);
4116 
4117 	mutex_exit(&igb->gen_lock);
4118 
4119 	if (link_changed)
4120 		mac_link_update(igb->mac_hdl, igb->link_state);
4121 
4122 	igb_start_watchdog_timer(igb);
4123 }
4124 
4125 /*
4126  * igb_intr_legacy - Interrupt handler for legacy interrupts
4127  */
4128 static uint_t
4129 igb_intr_legacy(void *arg1, void *arg2)
4130 {
4131 	igb_t *igb = (igb_t *)arg1;
4132 	igb_tx_ring_t *tx_ring;
4133 	uint32_t icr;
4134 	mblk_t *mp;
4135 	boolean_t tx_reschedule;
4136 	boolean_t link_changed;
4137 	uint_t result;
4138 
4139 	_NOTE(ARGUNUSED(arg2));
4140 
4141 	mutex_enter(&igb->gen_lock);
4142 
4143 	if (igb->igb_state & IGB_SUSPENDED) {
4144 		mutex_exit(&igb->gen_lock);
4145 		return (DDI_INTR_UNCLAIMED);
4146 	}
4147 
4148 	mp = NULL;
4149 	tx_reschedule = B_FALSE;
4150 	link_changed = B_FALSE;
4151 	icr = E1000_READ_REG(&igb->hw, E1000_ICR);
4152 
4153 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
4154 		mutex_exit(&igb->gen_lock);
4155 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
4156 		atomic_or_32(&igb->igb_state, IGB_ERROR);
4157 		return (DDI_INTR_UNCLAIMED);
4158 	}
4159 
4160 	if (icr & E1000_ICR_INT_ASSERTED) {
4161 		/*
4162 		 * E1000_ICR_INT_ASSERTED bit was set:
4163 		 * Read(Clear) the ICR, claim this interrupt,
4164 		 * look for work to do.
4165 		 */
4166 		ASSERT(igb->num_rx_rings == 1);
4167 		ASSERT(igb->num_tx_rings == 1);
4168 
4169 		/* Make sure all interrupt causes cleared */
4170 		(void) E1000_READ_REG(&igb->hw, E1000_EICR);
4171 
4172 		if (icr & E1000_ICR_RXT0) {
4173 			mp = igb_rx(&igb->rx_rings[0], IGB_NO_POLL);
4174 		}
4175 
4176 		if (icr & E1000_ICR_TXDW) {
4177 			tx_ring = &igb->tx_rings[0];
4178 
4179 			/* Recycle the tx descriptors */
4180 			tx_ring->tx_recycle(tx_ring);
4181 
4182 			/* Schedule the re-transmit */
4183 			tx_reschedule = (tx_ring->reschedule &&
4184 			    (tx_ring->tbd_free >= igb->tx_resched_thresh));
4185 		}
4186 
4187 		if (icr & E1000_ICR_LSC) {
4188 			/*
4189 			 * Because we got a link-status-change interrupt, force
4190 			 * e1000_check_for_link() to look at phy
4191 			 */
4192 			igb->hw.mac.get_link_status = B_TRUE;
4193 
4194 			/* igb_link_check takes care of link status change */
4195 			link_changed = igb_link_check(igb);
4196 
4197 			/* Get new phy state */
4198 			igb_get_phy_state(igb);
4199 		}
4200 
4201 		if (icr & E1000_ICR_DRSTA) {
4202 			/* 82580 Full Device Reset needed */
4203 			atomic_or_32(&igb->igb_state, IGB_STALL);
4204 		}
4205 
4206 		result = DDI_INTR_CLAIMED;
4207 	} else {
4208 		/*
4209 		 * E1000_ICR_INT_ASSERTED bit was not set:
4210 		 * Don't claim this interrupt.
4211 		 */
4212 		result = DDI_INTR_UNCLAIMED;
4213 	}
4214 
4215 	mutex_exit(&igb->gen_lock);
4216 
4217 	/*
4218 	 * Do the following work outside of the gen_lock
4219 	 */
4220 	if (mp != NULL)
4221 		mac_rx(igb->mac_hdl, NULL, mp);
4222 
4223 	if (tx_reschedule)  {
4224 		tx_ring->reschedule = B_FALSE;
4225 		mac_tx_ring_update(igb->mac_hdl, tx_ring->ring_handle);
4226 		IGB_DEBUG_STAT(tx_ring->stat_reschedule);
4227 	}
4228 
4229 	if (link_changed)
4230 		mac_link_update(igb->mac_hdl, igb->link_state);
4231 
4232 	return (result);
4233 }
4234 
4235 /*
4236  * igb_intr_msi - Interrupt handler for MSI
4237  */
4238 static uint_t
4239 igb_intr_msi(void *arg1, void *arg2)
4240 {
4241 	igb_t *igb = (igb_t *)arg1;
4242 	uint32_t icr;
4243 
4244 	_NOTE(ARGUNUSED(arg2));
4245 
4246 	icr = E1000_READ_REG(&igb->hw, E1000_ICR);
4247 
4248 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
4249 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
4250 		atomic_or_32(&igb->igb_state, IGB_ERROR);
4251 		return (DDI_INTR_CLAIMED);
4252 	}
4253 
4254 	/* Make sure all interrupt causes cleared */
4255 	(void) E1000_READ_REG(&igb->hw, E1000_EICR);
4256 
4257 	/*
4258 	 * For MSI interrupt, we have only one vector,
4259 	 * so we have only one rx ring and one tx ring enabled.
4260 	 */
4261 	ASSERT(igb->num_rx_rings == 1);
4262 	ASSERT(igb->num_tx_rings == 1);
4263 
4264 	if (icr & E1000_ICR_RXT0) {
4265 		igb_intr_rx_work(&igb->rx_rings[0]);
4266 	}
4267 
4268 	if (icr & E1000_ICR_TXDW) {
4269 		igb_intr_tx_work(&igb->tx_rings[0]);
4270 	}
4271 
4272 	if (icr & E1000_ICR_LSC) {
4273 		igb_intr_link_work(igb);
4274 	}
4275 
4276 	if (icr & E1000_ICR_DRSTA) {
4277 		/* 82580 Full Device Reset needed */
4278 		atomic_or_32(&igb->igb_state, IGB_STALL);
4279 	}
4280 
4281 	return (DDI_INTR_CLAIMED);
4282 }
4283 
4284 /*
4285  * igb_intr_rx - Interrupt handler for rx
4286  */
4287 static uint_t
4288 igb_intr_rx(void *arg1, void *arg2)
4289 {
4290 	igb_rx_ring_t *rx_ring = (igb_rx_ring_t *)arg1;
4291 
4292 	_NOTE(ARGUNUSED(arg2));
4293 
4294 	/*
4295 	 * Only used via MSI-X vector so don't check cause bits
4296 	 * and only clean the given ring.
4297 	 */
4298 	igb_intr_rx_work(rx_ring);
4299 
4300 	return (DDI_INTR_CLAIMED);
4301 }
4302 
4303 /*
4304  * igb_intr_tx - Interrupt handler for tx
4305  */
4306 static uint_t
4307 igb_intr_tx(void *arg1, void *arg2)
4308 {
4309 	igb_tx_ring_t *tx_ring = (igb_tx_ring_t *)arg1;
4310 
4311 	_NOTE(ARGUNUSED(arg2));
4312 
4313 	/*
4314 	 * Only used via MSI-X vector so don't check cause bits
4315 	 * and only clean the given ring.
4316 	 */
4317 	igb_intr_tx_work(tx_ring);
4318 
4319 	return (DDI_INTR_CLAIMED);
4320 }
4321 
4322 /*
4323  * igb_intr_tx_other - Interrupt handler for both tx and other
4324  *
4325  */
4326 static uint_t
4327 igb_intr_tx_other(void *arg1, void *arg2)
4328 {
4329 	igb_t *igb = (igb_t *)arg1;
4330 	uint32_t icr;
4331 
4332 	_NOTE(ARGUNUSED(arg2));
4333 
4334 	icr = E1000_READ_REG(&igb->hw, E1000_ICR);
4335 
4336 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
4337 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
4338 		atomic_or_32(&igb->igb_state, IGB_ERROR);
4339 		return (DDI_INTR_CLAIMED);
4340 	}
4341 
4342 	/*
4343 	 * Look for tx reclaiming work first. Remember, in the
4344 	 * case of only interrupt sharing, only one tx ring is
4345 	 * used
4346 	 */
4347 	igb_intr_tx_work(&igb->tx_rings[0]);
4348 
4349 	/*
4350 	 * Check for "other" causes.
4351 	 */
4352 	if (icr & E1000_ICR_LSC) {
4353 		igb_intr_link_work(igb);
4354 	}
4355 
4356 	/*
4357 	 * The DOUTSYNC bit indicates a tx packet dropped because
4358 	 * DMA engine gets "out of sync". There isn't a real fix
4359 	 * for this. The Intel recommendation is to count the number
4360 	 * of occurrences so user can detect when it is happening.
4361 	 * The issue is non-fatal and there's no recovery action
4362 	 * available.
4363 	 */
4364 	if (icr & E1000_ICR_DOUTSYNC) {
4365 		IGB_STAT(igb->dout_sync);
4366 	}
4367 
4368 	if (icr & E1000_ICR_DRSTA) {
4369 		/* 82580 Full Device Reset needed */
4370 		atomic_or_32(&igb->igb_state, IGB_STALL);
4371 	}
4372 
4373 	return (DDI_INTR_CLAIMED);
4374 }
4375 
4376 /*
4377  * igb_alloc_intrs - Allocate interrupts for the driver
4378  *
4379  * Normal sequence is to try MSI-X; if not sucessful, try MSI;
4380  * if not successful, try Legacy.
4381  * igb->intr_force can be used to force sequence to start with
4382  * any of the 3 types.
4383  * If MSI-X is not used, number of tx/rx rings is forced to 1.
4384  */
4385 static int
4386 igb_alloc_intrs(igb_t *igb)
4387 {
4388 	dev_info_t *devinfo;
4389 	int intr_types;
4390 	int rc;
4391 
4392 	devinfo = igb->dip;
4393 
4394 	/* Get supported interrupt types */
4395 	rc = ddi_intr_get_supported_types(devinfo, &intr_types);
4396 
4397 	if (rc != DDI_SUCCESS) {
4398 		igb_log(igb, IGB_LOG_ERROR,
4399 		    "Get supported interrupt types failed: %d", rc);
4400 		return (IGB_FAILURE);
4401 	}
4402 	igb_log(igb, IGB_LOG_INFO, "Supported interrupt types: %x",
4403 	    intr_types);
4404 
4405 	igb->intr_type = 0;
4406 
4407 	/* Install MSI-X interrupts */
4408 	if ((intr_types & DDI_INTR_TYPE_MSIX) &&
4409 	    (igb->intr_force <= IGB_INTR_MSIX)) {
4410 		rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_MSIX);
4411 
4412 		if (rc == IGB_SUCCESS)
4413 			return (IGB_SUCCESS);
4414 
4415 		igb_log(igb, IGB_LOG_INFO,
4416 		    "Allocate MSI-X failed, trying MSI interrupts...");
4417 	}
4418 
4419 	/* MSI-X not used, force rings to 1 */
4420 	igb->num_rx_rings = 1;
4421 	igb->num_tx_rings = 1;
4422 	igb_log(igb, IGB_LOG_INFO,
4423 	    "MSI-X not used, force rx and tx queue number to 1");
4424 
4425 	/* Install MSI interrupts */
4426 	if ((intr_types & DDI_INTR_TYPE_MSI) &&
4427 	    (igb->intr_force <= IGB_INTR_MSI)) {
4428 		rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_MSI);
4429 
4430 		if (rc == IGB_SUCCESS)
4431 			return (IGB_SUCCESS);
4432 
4433 		igb_log(igb, IGB_LOG_INFO,
4434 		    "Allocate MSI failed, trying Legacy interrupts...");
4435 	}
4436 
4437 	/* Install legacy interrupts */
4438 	if (intr_types & DDI_INTR_TYPE_FIXED) {
4439 		rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_FIXED);
4440 
4441 		if (rc == IGB_SUCCESS)
4442 			return (IGB_SUCCESS);
4443 
4444 		igb_log(igb, IGB_LOG_INFO,
4445 		    "Allocate Legacy interrupts failed");
4446 	}
4447 
4448 	/* If none of the 3 types succeeded, return failure */
4449 	return (IGB_FAILURE);
4450 }
4451 
4452 /*
4453  * igb_alloc_intr_handles - Allocate interrupt handles.
4454  *
4455  * For legacy and MSI, only 1 handle is needed.  For MSI-X,
4456  * if fewer than 2 handles are available, return failure.
4457  * Upon success, this sets the number of Rx rings to a number that
4458  * matches the handles available for Rx interrupts.
4459  */
4460 static int
4461 igb_alloc_intr_handles(igb_t *igb, int intr_type)
4462 {
4463 	dev_info_t *devinfo;
4464 	int orig, request, count, avail, actual;
4465 	int diff, minimum;
4466 	int rc;
4467 
4468 	devinfo = igb->dip;
4469 
4470 	switch (intr_type) {
4471 	case DDI_INTR_TYPE_FIXED:
4472 		request = 1;	/* Request 1 legacy interrupt handle */
4473 		minimum = 1;
4474 		igb_log(igb, IGB_LOG_INFO, "interrupt type: legacy");
4475 		break;
4476 
4477 	case DDI_INTR_TYPE_MSI:
4478 		request = 1;	/* Request 1 MSI interrupt handle */
4479 		minimum = 1;
4480 		igb_log(igb, IGB_LOG_INFO, "interrupt type: MSI");
4481 		break;
4482 
4483 	case DDI_INTR_TYPE_MSIX:
4484 		/*
4485 		 * Number of vectors for the adapter is
4486 		 * # rx rings + # tx rings
4487 		 * One of tx vectors is for tx & other
4488 		 */
4489 		request = igb->num_rx_rings + igb->num_tx_rings;
4490 		orig = request;
4491 		minimum = 2;
4492 		igb_log(igb, IGB_LOG_INFO, "interrupt type: MSI-X");
4493 		break;
4494 
4495 	default:
4496 		igb_log(igb, IGB_LOG_INFO,
4497 		    "invalid call to igb_alloc_intr_handles(): %d\n",
4498 		    intr_type);
4499 		return (IGB_FAILURE);
4500 	}
4501 	igb_log(igb, IGB_LOG_INFO,
4502 	    "interrupt handles requested: %d  minimum: %d",
4503 	    request, minimum);
4504 
4505 	/*
4506 	 * Get number of supported interrupts
4507 	 */
4508 	rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
4509 	if ((rc != DDI_SUCCESS) || (count < minimum)) {
4510 		igb_log(igb, IGB_LOG_INFO,
4511 		    "Get supported interrupt number failed. "
4512 		    "Return: %d, count: %d", rc, count);
4513 		return (IGB_FAILURE);
4514 	}
4515 	igb_log(igb, IGB_LOG_INFO, "interrupts supported: %d", count);
4516 
4517 	/*
4518 	 * Get number of available interrupts
4519 	 */
4520 	rc = ddi_intr_get_navail(devinfo, intr_type, &avail);
4521 	if ((rc != DDI_SUCCESS) || (avail < minimum)) {
4522 		igb_log(igb, IGB_LOG_INFO,
4523 		    "Get available interrupt number failed. "
4524 		    "Return: %d, available: %d", rc, avail);
4525 		return (IGB_FAILURE);
4526 	}
4527 	igb_log(igb, IGB_LOG_INFO, "interrupts available: %d", avail);
4528 
4529 	if (avail < request) {
4530 		igb_log(igb, IGB_LOG_INFO,
4531 		    "Request %d handles, %d available",
4532 		    request, avail);
4533 		request = avail;
4534 	}
4535 
4536 	actual = 0;
4537 	igb->intr_cnt = 0;
4538 
4539 	/*
4540 	 * Allocate an array of interrupt handles
4541 	 */
4542 	igb->intr_size = request * sizeof (ddi_intr_handle_t);
4543 	igb->htable = kmem_alloc(igb->intr_size, KM_SLEEP);
4544 
4545 	rc = ddi_intr_alloc(devinfo, igb->htable, intr_type, 0,
4546 	    request, &actual, DDI_INTR_ALLOC_NORMAL);
4547 	if (rc != DDI_SUCCESS) {
4548 		igb_log(igb, IGB_LOG_INFO, "Allocate interrupts failed. "
4549 		    "return: %d, request: %d, actual: %d",
4550 		    rc, request, actual);
4551 		goto alloc_handle_fail;
4552 	}
4553 	igb_log(igb, IGB_LOG_INFO, "interrupts actually allocated: %d", actual);
4554 
4555 	igb->intr_cnt = actual;
4556 
4557 	if (actual < minimum) {
4558 		igb_log(igb, IGB_LOG_INFO,
4559 		    "Insufficient interrupt handles allocated: %d",
4560 		    actual);
4561 		goto alloc_handle_fail;
4562 	}
4563 
4564 	/*
4565 	 * For MSI-X, actual might force us to reduce number of tx & rx rings
4566 	 */
4567 	if ((intr_type == DDI_INTR_TYPE_MSIX) && (orig > actual)) {
4568 		diff = orig - actual;
4569 		if (diff < igb->num_tx_rings) {
4570 			igb_log(igb, IGB_LOG_INFO,
4571 			    "MSI-X vectors force Tx queue number to %d",
4572 			    igb->num_tx_rings - diff);
4573 			igb->num_tx_rings -= diff;
4574 		} else {
4575 			igb_log(igb, IGB_LOG_INFO,
4576 			    "MSI-X vectors force Tx queue number to 1");
4577 			igb->num_tx_rings = 1;
4578 
4579 			igb_log(igb, IGB_LOG_INFO,
4580 			    "MSI-X vectors force Rx queue number to %d",
4581 			    actual - 1);
4582 			igb->num_rx_rings = actual - 1;
4583 		}
4584 	}
4585 
4586 	/*
4587 	 * Get priority for first vector, assume remaining are all the same
4588 	 */
4589 	rc = ddi_intr_get_pri(igb->htable[0], &igb->intr_pri);
4590 	if (rc != DDI_SUCCESS) {
4591 		igb_log(igb, IGB_LOG_INFO,
4592 		    "Get interrupt priority failed: %d", rc);
4593 		goto alloc_handle_fail;
4594 	}
4595 
4596 	rc = ddi_intr_get_cap(igb->htable[0], &igb->intr_cap);
4597 	if (rc != DDI_SUCCESS) {
4598 		igb_log(igb, IGB_LOG_INFO,
4599 		    "Get interrupt cap failed: %d", rc);
4600 		goto alloc_handle_fail;
4601 	}
4602 
4603 	igb->intr_type = intr_type;
4604 
4605 	return (IGB_SUCCESS);
4606 
4607 alloc_handle_fail:
4608 	igb_rem_intrs(igb);
4609 
4610 	return (IGB_FAILURE);
4611 }
4612 
4613 /*
4614  * igb_add_intr_handlers - Add interrupt handlers based on the interrupt type
4615  *
4616  * Before adding the interrupt handlers, the interrupt vectors have
4617  * been allocated, and the rx/tx rings have also been allocated.
4618  */
4619 static int
4620 igb_add_intr_handlers(igb_t *igb)
4621 {
4622 	igb_rx_ring_t *rx_ring;
4623 	igb_tx_ring_t *tx_ring;
4624 	int vector;
4625 	int rc;
4626 	int i;
4627 
4628 	vector = 0;
4629 
4630 	switch (igb->intr_type) {
4631 	case DDI_INTR_TYPE_MSIX:
4632 		/* Add interrupt handler for tx + other */
4633 		tx_ring = &igb->tx_rings[0];
4634 		rc = ddi_intr_add_handler(igb->htable[vector],
4635 		    (ddi_intr_handler_t *)igb_intr_tx_other,
4636 		    (void *)igb, NULL);
4637 
4638 		if (rc != DDI_SUCCESS) {
4639 			igb_log(igb, IGB_LOG_INFO,
4640 			    "Add tx/other interrupt handler failed: %d", rc);
4641 			return (IGB_FAILURE);
4642 		}
4643 		tx_ring->intr_vector = vector;
4644 		vector++;
4645 
4646 		/* Add interrupt handler for each rx ring */
4647 		for (i = 0; i < igb->num_rx_rings; i++) {
4648 			rx_ring = &igb->rx_rings[i];
4649 
4650 			rc = ddi_intr_add_handler(igb->htable[vector],
4651 			    (ddi_intr_handler_t *)igb_intr_rx,
4652 			    (void *)rx_ring, NULL);
4653 
4654 			if (rc != DDI_SUCCESS) {
4655 				igb_log(igb, IGB_LOG_INFO,
4656 				    "Add rx interrupt handler failed. "
4657 				    "return: %d, rx ring: %d", rc, i);
4658 				for (vector--; vector >= 0; vector--) {
4659 					(void) ddi_intr_remove_handler(
4660 					    igb->htable[vector]);
4661 				}
4662 				return (IGB_FAILURE);
4663 			}
4664 
4665 			rx_ring->intr_vector = vector;
4666 
4667 			vector++;
4668 		}
4669 
4670 		/* Add interrupt handler for each tx ring from 2nd ring */
4671 		for (i = 1; i < igb->num_tx_rings; i++) {
4672 			tx_ring = &igb->tx_rings[i];
4673 
4674 			rc = ddi_intr_add_handler(igb->htable[vector],
4675 			    (ddi_intr_handler_t *)igb_intr_tx,
4676 			    (void *)tx_ring, NULL);
4677 
4678 			if (rc != DDI_SUCCESS) {
4679 				igb_log(igb, IGB_LOG_INFO,
4680 				    "Add tx interrupt handler failed. "
4681 				    "return: %d, tx ring: %d", rc, i);
4682 				for (vector--; vector >= 0; vector--) {
4683 					(void) ddi_intr_remove_handler(
4684 					    igb->htable[vector]);
4685 				}
4686 				return (IGB_FAILURE);
4687 			}
4688 
4689 			tx_ring->intr_vector = vector;
4690 
4691 			vector++;
4692 		}
4693 
4694 		break;
4695 
4696 	case DDI_INTR_TYPE_MSI:
4697 		/* Add interrupt handlers for the only vector */
4698 		rc = ddi_intr_add_handler(igb->htable[vector],
4699 		    (ddi_intr_handler_t *)igb_intr_msi,
4700 		    (void *)igb, NULL);
4701 
4702 		if (rc != DDI_SUCCESS) {
4703 			igb_log(igb, IGB_LOG_INFO,
4704 			    "Add MSI interrupt handler failed: %d", rc);
4705 			return (IGB_FAILURE);
4706 		}
4707 
4708 		rx_ring = &igb->rx_rings[0];
4709 		rx_ring->intr_vector = vector;
4710 
4711 		vector++;
4712 		break;
4713 
4714 	case DDI_INTR_TYPE_FIXED:
4715 		/* Add interrupt handlers for the only vector */
4716 		rc = ddi_intr_add_handler(igb->htable[vector],
4717 		    (ddi_intr_handler_t *)igb_intr_legacy,
4718 		    (void *)igb, NULL);
4719 
4720 		if (rc != DDI_SUCCESS) {
4721 			igb_log(igb, IGB_LOG_INFO,
4722 			    "Add legacy interrupt handler failed: %d", rc);
4723 			return (IGB_FAILURE);
4724 		}
4725 
4726 		rx_ring = &igb->rx_rings[0];
4727 		rx_ring->intr_vector = vector;
4728 
4729 		vector++;
4730 		break;
4731 
4732 	default:
4733 		return (IGB_FAILURE);
4734 	}
4735 
4736 	ASSERT(vector == igb->intr_cnt);
4737 
4738 	return (IGB_SUCCESS);
4739 }
4740 
4741 /*
4742  * igb_setup_msix_82575 - setup 82575 adapter to use MSI-X interrupts
4743  *
4744  * For each vector enabled on the adapter, Set the MSIXBM register accordingly
4745  */
4746 static void
4747 igb_setup_msix_82575(igb_t *igb)
4748 {
4749 	uint32_t eims = 0;
4750 	int i, vector;
4751 	struct e1000_hw *hw = &igb->hw;
4752 
4753 	/*
4754 	 * Set vector for tx ring 0 and other causes.
4755 	 * NOTE assumption that it is vector 0.
4756 	 */
4757 	vector = 0;
4758 
4759 	igb->eims_mask = E1000_EICR_TX_QUEUE0 | E1000_EICR_OTHER;
4760 	E1000_WRITE_REG(hw, E1000_MSIXBM(vector), igb->eims_mask);
4761 	vector++;
4762 
4763 	for (i = 0; i < igb->num_rx_rings; i++) {
4764 		/*
4765 		 * Set vector for each rx ring
4766 		 */
4767 		eims = (E1000_EICR_RX_QUEUE0 << i);
4768 		E1000_WRITE_REG(hw, E1000_MSIXBM(vector), eims);
4769 
4770 		/*
4771 		 * Accumulate bits to enable in
4772 		 * igb_enable_adapter_interrupts_82575()
4773 		 */
4774 		igb->eims_mask |= eims;
4775 
4776 		vector++;
4777 	}
4778 
4779 	for (i = 1; i < igb->num_tx_rings; i++) {
4780 		/*
4781 		 * Set vector for each tx ring from 2nd tx ring
4782 		 */
4783 		eims = (E1000_EICR_TX_QUEUE0 << i);
4784 		E1000_WRITE_REG(hw, E1000_MSIXBM(vector), eims);
4785 
4786 		/*
4787 		 * Accumulate bits to enable in
4788 		 * igb_enable_adapter_interrupts_82575()
4789 		 */
4790 		igb->eims_mask |= eims;
4791 
4792 		vector++;
4793 	}
4794 
4795 	ASSERT(vector == igb->intr_cnt);
4796 
4797 	/*
4798 	 * Disable IAM for ICR interrupt bits
4799 	 */
4800 	E1000_WRITE_REG(hw, E1000_IAM, 0);
4801 	E1000_WRITE_FLUSH(hw);
4802 }
4803 
4804 /*
4805  * igb_setup_msix_82576 - setup 82576 adapter to use MSI-X interrupts
4806  *
4807  * 82576 uses a table based method for assigning vectors.  Each queue has a
4808  * single entry in the table to which we write a vector number along with a
4809  * "valid" bit.  The entry is a single byte in a 4-byte register.  Vectors
4810  * take a different position in the 4-byte register depending on whether
4811  * they are numbered above or below 8.
4812  */
4813 static void
4814 igb_setup_msix_82576(igb_t *igb)
4815 {
4816 	struct e1000_hw *hw = &igb->hw;
4817 	uint32_t ivar, index, vector;
4818 	int i;
4819 
4820 	/* must enable msi-x capability before IVAR settings */
4821 	E1000_WRITE_REG(hw, E1000_GPIE,
4822 	    (E1000_GPIE_MSIX_MODE | E1000_GPIE_PBA | E1000_GPIE_NSICR));
4823 
4824 	/*
4825 	 * Set vector for tx ring 0 and other causes.
4826 	 * NOTE assumption that it is vector 0.
4827 	 * This is also interdependent with installation of interrupt service
4828 	 * routines in igb_add_intr_handlers().
4829 	 */
4830 
4831 	/* assign "other" causes to vector 0 */
4832 	vector = 0;
4833 	ivar = ((vector | E1000_IVAR_VALID) << 8);
4834 	E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
4835 
4836 	/* assign tx ring 0 to vector 0 */
4837 	ivar = ((vector | E1000_IVAR_VALID) << 8);
4838 	E1000_WRITE_REG(hw, E1000_IVAR0, ivar);
4839 
4840 	/* prepare to enable tx & other interrupt causes */
4841 	igb->eims_mask = (1 << vector);
4842 
4843 	vector ++;
4844 	for (i = 0; i < igb->num_rx_rings; i++) {
4845 		/*
4846 		 * Set vector for each rx ring
4847 		 */
4848 		index = (i & 0x7);
4849 		ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4850 
4851 		if (i < 8) {
4852 			/* vector goes into low byte of register */
4853 			ivar = ivar & 0xFFFFFF00;
4854 			ivar |= (vector | E1000_IVAR_VALID);
4855 		} else {
4856 			/* vector goes into third byte of register */
4857 			ivar = ivar & 0xFF00FFFF;
4858 			ivar |= ((vector | E1000_IVAR_VALID) << 16);
4859 		}
4860 		E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
4861 
4862 		/* Accumulate interrupt-cause bits to enable */
4863 		igb->eims_mask |= (1 << vector);
4864 
4865 		vector ++;
4866 	}
4867 
4868 	for (i = 1; i < igb->num_tx_rings; i++) {
4869 		/*
4870 		 * Set vector for each tx ring from 2nd tx ring.
4871 		 * Note assumption that tx vectors numericall follow rx vectors.
4872 		 */
4873 		index = (i & 0x7);
4874 		ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4875 
4876 		if (i < 8) {
4877 			/* vector goes into second byte of register */
4878 			ivar = ivar & 0xFFFF00FF;
4879 			ivar |= ((vector | E1000_IVAR_VALID) << 8);
4880 		} else {
4881 			/* vector goes into fourth byte of register */
4882 			ivar = ivar & 0x00FFFFFF;
4883 			ivar |= (vector | E1000_IVAR_VALID) << 24;
4884 		}
4885 		E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
4886 
4887 		/* Accumulate interrupt-cause bits to enable */
4888 		igb->eims_mask |= (1 << vector);
4889 
4890 		vector ++;
4891 	}
4892 
4893 	ASSERT(vector == igb->intr_cnt);
4894 }
4895 
4896 /*
4897  * igb_setup_msix_82580 - setup 82580 adapter to use MSI-X interrupts
4898  *
4899  * 82580 uses same table approach at 82576 but has fewer entries.  Each
4900  * queue has a single entry in the table to which we write a vector number
4901  * along with a "valid" bit.  Vectors take a different position in the
4902  * register depending on * whether * they are numbered above or below 4.
4903  */
4904 static void
4905 igb_setup_msix_82580(igb_t *igb)
4906 {
4907 	struct e1000_hw *hw = &igb->hw;
4908 	uint32_t ivar, index, vector;
4909 	int i;
4910 
4911 	/* must enable msi-x capability before IVAR settings */
4912 	E1000_WRITE_REG(hw, E1000_GPIE, (E1000_GPIE_MSIX_MODE |
4913 	    E1000_GPIE_PBA | E1000_GPIE_NSICR | E1000_GPIE_EIAME));
4914 	/*
4915 	 * Set vector for tx ring 0 and other causes.
4916 	 * NOTE assumption that it is vector 0.
4917 	 * This is also interdependent with installation of interrupt service
4918 	 * routines in igb_add_intr_handlers().
4919 	 */
4920 
4921 	/* assign "other" causes to vector 0 */
4922 	vector = 0;
4923 	ivar = ((vector | E1000_IVAR_VALID) << 8);
4924 	E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
4925 
4926 	/* assign tx ring 0 to vector 0 */
4927 	ivar = ((vector | E1000_IVAR_VALID) << 8);
4928 	E1000_WRITE_REG(hw, E1000_IVAR0, ivar);
4929 
4930 	/* prepare to enable tx & other interrupt causes */
4931 	igb->eims_mask = (1 << vector);
4932 
4933 	vector ++;
4934 
4935 	for (i = 0; i < igb->num_rx_rings; i++) {
4936 		/*
4937 		 * Set vector for each rx ring
4938 		 */
4939 		index = (i >> 1);
4940 		ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4941 
4942 		if (i & 1) {
4943 			/* vector goes into third byte of register */
4944 			ivar = ivar & 0xFF00FFFF;
4945 			ivar |= ((vector | E1000_IVAR_VALID) << 16);
4946 		} else {
4947 			/* vector goes into low byte of register */
4948 			ivar = ivar & 0xFFFFFF00;
4949 			ivar |= (vector | E1000_IVAR_VALID);
4950 		}
4951 		E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
4952 
4953 		/* Accumulate interrupt-cause bits to enable */
4954 		igb->eims_mask |= (1 << vector);
4955 
4956 		vector ++;
4957 	}
4958 
4959 	for (i = 1; i < igb->num_tx_rings; i++) {
4960 		/*
4961 		 * Set vector for each tx ring from 2nd tx ring.
4962 		 * Note assumption that tx vectors numericall follow rx vectors.
4963 		 */
4964 		index = (i >> 1);
4965 		ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4966 
4967 		if (i & 1) {
4968 			/* vector goes into high byte of register */
4969 			ivar = ivar & 0x00FFFFFF;
4970 			ivar |= ((vector | E1000_IVAR_VALID) << 24);
4971 		} else {
4972 			/* vector goes into second byte of register */
4973 			ivar = ivar & 0xFFFF00FF;
4974 			ivar |= (vector | E1000_IVAR_VALID) << 8;
4975 		}
4976 		E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
4977 
4978 		/* Accumulate interrupt-cause bits to enable */
4979 		igb->eims_mask |= (1 << vector);
4980 
4981 		vector ++;
4982 	}
4983 	ASSERT(vector == igb->intr_cnt);
4984 }
4985 
4986 /*
4987  * igb_rem_intr_handlers - remove the interrupt handlers
4988  */
4989 static void
4990 igb_rem_intr_handlers(igb_t *igb)
4991 {
4992 	int i;
4993 	int rc;
4994 
4995 	for (i = 0; i < igb->intr_cnt; i++) {
4996 		rc = ddi_intr_remove_handler(igb->htable[i]);
4997 		if (rc != DDI_SUCCESS) {
4998 			igb_log(igb, IGB_LOG_INFO,
4999 			    "Remove intr handler failed: %d", rc);
5000 		}
5001 	}
5002 }
5003 
5004 /*
5005  * igb_rem_intrs - remove the allocated interrupts
5006  */
5007 static void
5008 igb_rem_intrs(igb_t *igb)
5009 {
5010 	int i;
5011 	int rc;
5012 
5013 	for (i = 0; i < igb->intr_cnt; i++) {
5014 		rc = ddi_intr_free(igb->htable[i]);
5015 		if (rc != DDI_SUCCESS) {
5016 			igb_log(igb, IGB_LOG_INFO,
5017 			    "Free intr failed: %d", rc);
5018 		}
5019 	}
5020 
5021 	kmem_free(igb->htable, igb->intr_size);
5022 	igb->htable = NULL;
5023 }
5024 
5025 /*
5026  * igb_enable_intrs - enable all the ddi interrupts
5027  */
5028 static int
5029 igb_enable_intrs(igb_t *igb)
5030 {
5031 	int i;
5032 	int rc;
5033 
5034 	/* Enable interrupts */
5035 	if (igb->intr_cap & DDI_INTR_FLAG_BLOCK) {
5036 		/* Call ddi_intr_block_enable() for MSI */
5037 		rc = ddi_intr_block_enable(igb->htable, igb->intr_cnt);
5038 		if (rc != DDI_SUCCESS) {
5039 			igb_log(igb, IGB_LOG_ERROR,
5040 			    "Enable block intr failed: %d", rc);
5041 			return (IGB_FAILURE);
5042 		}
5043 	} else {
5044 		/* Call ddi_intr_enable() for Legacy/MSI non block enable */
5045 		for (i = 0; i < igb->intr_cnt; i++) {
5046 			rc = ddi_intr_enable(igb->htable[i]);
5047 			if (rc != DDI_SUCCESS) {
5048 				igb_log(igb, IGB_LOG_ERROR,
5049 				    "Enable intr failed: %d", rc);
5050 				return (IGB_FAILURE);
5051 			}
5052 		}
5053 	}
5054 
5055 	return (IGB_SUCCESS);
5056 }
5057 
5058 /*
5059  * igb_disable_intrs - disable all the ddi interrupts
5060  */
5061 static int
5062 igb_disable_intrs(igb_t *igb)
5063 {
5064 	int i;
5065 	int rc;
5066 
5067 	/* Disable all interrupts */
5068 	if (igb->intr_cap & DDI_INTR_FLAG_BLOCK) {
5069 		rc = ddi_intr_block_disable(igb->htable, igb->intr_cnt);
5070 		if (rc != DDI_SUCCESS) {
5071 			igb_log(igb, IGB_LOG_ERROR,
5072 			    "Disable block intr failed: %d", rc);
5073 			return (IGB_FAILURE);
5074 		}
5075 	} else {
5076 		for (i = 0; i < igb->intr_cnt; i++) {
5077 			rc = ddi_intr_disable(igb->htable[i]);
5078 			if (rc != DDI_SUCCESS) {
5079 				igb_log(igb, IGB_LOG_ERROR,
5080 				    "Disable intr failed: %d", rc);
5081 				return (IGB_FAILURE);
5082 			}
5083 		}
5084 	}
5085 
5086 	return (IGB_SUCCESS);
5087 }
5088 
5089 /*
5090  * igb_get_phy_state - Get and save the parameters read from PHY registers
5091  */
5092 static void
5093 igb_get_phy_state(igb_t *igb)
5094 {
5095 	struct e1000_hw *hw = &igb->hw;
5096 	uint16_t phy_ctrl;
5097 	uint16_t phy_status;
5098 	uint16_t phy_an_adv;
5099 	uint16_t phy_an_exp;
5100 	uint16_t phy_ext_status;
5101 	uint16_t phy_1000t_ctrl;
5102 	uint16_t phy_1000t_status;
5103 	uint16_t phy_lp_able;
5104 
5105 	ASSERT(mutex_owned(&igb->gen_lock));
5106 
5107 	if (hw->phy.media_type == e1000_media_type_copper) {
5108 		(void) e1000_read_phy_reg(hw, PHY_CONTROL, &phy_ctrl);
5109 		(void) e1000_read_phy_reg(hw, PHY_STATUS, &phy_status);
5110 		(void) e1000_read_phy_reg(hw, PHY_AUTONEG_ADV, &phy_an_adv);
5111 		(void) e1000_read_phy_reg(hw, PHY_AUTONEG_EXP, &phy_an_exp);
5112 		(void) e1000_read_phy_reg(hw, PHY_EXT_STATUS, &phy_ext_status);
5113 		(void) e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_1000t_ctrl);
5114 		(void) e1000_read_phy_reg(hw,
5115 		    PHY_1000T_STATUS, &phy_1000t_status);
5116 		(void) e1000_read_phy_reg(hw, PHY_LP_ABILITY, &phy_lp_able);
5117 
5118 		igb->param_autoneg_cap =
5119 		    (phy_status & MII_SR_AUTONEG_CAPS) ? 1 : 0;
5120 		igb->param_pause_cap =
5121 		    (phy_an_adv & NWAY_AR_PAUSE) ? 1 : 0;
5122 		igb->param_asym_pause_cap =
5123 		    (phy_an_adv & NWAY_AR_ASM_DIR) ? 1 : 0;
5124 		igb->param_1000fdx_cap =
5125 		    ((phy_ext_status & IEEE_ESR_1000T_FD_CAPS) ||
5126 		    (phy_ext_status & IEEE_ESR_1000X_FD_CAPS)) ? 1 : 0;
5127 		igb->param_1000hdx_cap =
5128 		    ((phy_ext_status & IEEE_ESR_1000T_HD_CAPS) ||
5129 		    (phy_ext_status & IEEE_ESR_1000X_HD_CAPS)) ? 1 : 0;
5130 		igb->param_100t4_cap =
5131 		    (phy_status & MII_SR_100T4_CAPS) ? 1 : 0;
5132 		igb->param_100fdx_cap = ((phy_status & MII_SR_100X_FD_CAPS) ||
5133 		    (phy_status & MII_SR_100T2_FD_CAPS)) ? 1 : 0;
5134 		igb->param_100hdx_cap = ((phy_status & MII_SR_100X_HD_CAPS) ||
5135 		    (phy_status & MII_SR_100T2_HD_CAPS)) ? 1 : 0;
5136 		igb->param_10fdx_cap =
5137 		    (phy_status & MII_SR_10T_FD_CAPS) ? 1 : 0;
5138 		igb->param_10hdx_cap =
5139 		    (phy_status & MII_SR_10T_HD_CAPS) ? 1 : 0;
5140 		igb->param_rem_fault =
5141 		    (phy_status & MII_SR_REMOTE_FAULT) ? 1 : 0;
5142 
5143 		igb->param_adv_autoneg_cap = hw->mac.autoneg;
5144 		igb->param_adv_pause_cap =
5145 		    (phy_an_adv & NWAY_AR_PAUSE) ? 1 : 0;
5146 		igb->param_adv_asym_pause_cap =
5147 		    (phy_an_adv & NWAY_AR_ASM_DIR) ? 1 : 0;
5148 		igb->param_adv_1000hdx_cap =
5149 		    (phy_1000t_ctrl & CR_1000T_HD_CAPS) ? 1 : 0;
5150 		igb->param_adv_100t4_cap =
5151 		    (phy_an_adv & NWAY_AR_100T4_CAPS) ? 1 : 0;
5152 		igb->param_adv_rem_fault =
5153 		    (phy_an_adv & NWAY_AR_REMOTE_FAULT) ? 1 : 0;
5154 		if (igb->param_adv_autoneg_cap == 1) {
5155 			igb->param_adv_1000fdx_cap =
5156 			    (phy_1000t_ctrl & CR_1000T_FD_CAPS) ? 1 : 0;
5157 			igb->param_adv_100fdx_cap =
5158 			    (phy_an_adv & NWAY_AR_100TX_FD_CAPS) ? 1 : 0;
5159 			igb->param_adv_100hdx_cap =
5160 			    (phy_an_adv & NWAY_AR_100TX_HD_CAPS) ? 1 : 0;
5161 			igb->param_adv_10fdx_cap =
5162 			    (phy_an_adv & NWAY_AR_10T_FD_CAPS) ? 1 : 0;
5163 			igb->param_adv_10hdx_cap =
5164 			    (phy_an_adv & NWAY_AR_10T_HD_CAPS) ? 1 : 0;
5165 		}
5166 
5167 		igb->param_lp_autoneg_cap =
5168 		    (phy_an_exp & NWAY_ER_LP_NWAY_CAPS) ? 1 : 0;
5169 		igb->param_lp_pause_cap =
5170 		    (phy_lp_able & NWAY_LPAR_PAUSE) ? 1 : 0;
5171 		igb->param_lp_asym_pause_cap =
5172 		    (phy_lp_able & NWAY_LPAR_ASM_DIR) ? 1 : 0;
5173 		igb->param_lp_1000fdx_cap =
5174 		    (phy_1000t_status & SR_1000T_LP_FD_CAPS) ? 1 : 0;
5175 		igb->param_lp_1000hdx_cap =
5176 		    (phy_1000t_status & SR_1000T_LP_HD_CAPS) ? 1 : 0;
5177 		igb->param_lp_100t4_cap =
5178 		    (phy_lp_able & NWAY_LPAR_100T4_CAPS) ? 1 : 0;
5179 		igb->param_lp_100fdx_cap =
5180 		    (phy_lp_able & NWAY_LPAR_100TX_FD_CAPS) ? 1 : 0;
5181 		igb->param_lp_100hdx_cap =
5182 		    (phy_lp_able & NWAY_LPAR_100TX_HD_CAPS) ? 1 : 0;
5183 		igb->param_lp_10fdx_cap =
5184 		    (phy_lp_able & NWAY_LPAR_10T_FD_CAPS) ? 1 : 0;
5185 		igb->param_lp_10hdx_cap =
5186 		    (phy_lp_able & NWAY_LPAR_10T_HD_CAPS) ? 1 : 0;
5187 		igb->param_lp_rem_fault =
5188 		    (phy_lp_able & NWAY_LPAR_REMOTE_FAULT) ? 1 : 0;
5189 	} else {
5190 		/*
5191 		 * 1Gig Fiber adapter only offers 1Gig Full Duplex.
5192 		 */
5193 		igb->param_autoneg_cap = 0;
5194 		igb->param_pause_cap = 1;
5195 		igb->param_asym_pause_cap = 1;
5196 		igb->param_1000fdx_cap = 1;
5197 		igb->param_1000hdx_cap = 0;
5198 		igb->param_100t4_cap = 0;
5199 		igb->param_100fdx_cap = 0;
5200 		igb->param_100hdx_cap = 0;
5201 		igb->param_10fdx_cap = 0;
5202 		igb->param_10hdx_cap = 0;
5203 
5204 		igb->param_adv_autoneg_cap = 0;
5205 		igb->param_adv_pause_cap = 1;
5206 		igb->param_adv_asym_pause_cap = 1;
5207 		igb->param_adv_1000fdx_cap = 1;
5208 		igb->param_adv_1000hdx_cap = 0;
5209 		igb->param_adv_100t4_cap = 0;
5210 		igb->param_adv_100fdx_cap = 0;
5211 		igb->param_adv_100hdx_cap = 0;
5212 		igb->param_adv_10fdx_cap = 0;
5213 		igb->param_adv_10hdx_cap = 0;
5214 
5215 		igb->param_lp_autoneg_cap = 0;
5216 		igb->param_lp_pause_cap = 0;
5217 		igb->param_lp_asym_pause_cap = 0;
5218 		igb->param_lp_1000fdx_cap = 0;
5219 		igb->param_lp_1000hdx_cap = 0;
5220 		igb->param_lp_100t4_cap = 0;
5221 		igb->param_lp_100fdx_cap = 0;
5222 		igb->param_lp_100hdx_cap = 0;
5223 		igb->param_lp_10fdx_cap = 0;
5224 		igb->param_lp_10hdx_cap = 0;
5225 		igb->param_lp_rem_fault = 0;
5226 	}
5227 }
5228 
5229 /*
5230  * synchronize the adv* and en* parameters.
5231  *
5232  * See comments in <sys/dld.h> for details of the *_en_*
5233  * parameters. The usage of ndd for setting adv parameters will
5234  * synchronize all the en parameters with the e1000g parameters,
5235  * implicitly disabling any settings made via dladm.
5236  */
5237 static void
5238 igb_param_sync(igb_t *igb)
5239 {
5240 	igb->param_en_1000fdx_cap = igb->param_adv_1000fdx_cap;
5241 	igb->param_en_1000hdx_cap = igb->param_adv_1000hdx_cap;
5242 	igb->param_en_100t4_cap = igb->param_adv_100t4_cap;
5243 	igb->param_en_100fdx_cap = igb->param_adv_100fdx_cap;
5244 	igb->param_en_100hdx_cap = igb->param_adv_100hdx_cap;
5245 	igb->param_en_10fdx_cap = igb->param_adv_10fdx_cap;
5246 	igb->param_en_10hdx_cap = igb->param_adv_10hdx_cap;
5247 }
5248 
5249 /*
5250  * igb_get_driver_control
5251  */
5252 static void
5253 igb_get_driver_control(struct e1000_hw *hw)
5254 {
5255 	uint32_t ctrl_ext;
5256 
5257 	/* Notify firmware that driver is in control of device */
5258 	ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
5259 	ctrl_ext |= E1000_CTRL_EXT_DRV_LOAD;
5260 	E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
5261 }
5262 
5263 /*
5264  * igb_release_driver_control
5265  */
5266 static void
5267 igb_release_driver_control(struct e1000_hw *hw)
5268 {
5269 	uint32_t ctrl_ext;
5270 
5271 	/* Notify firmware that driver is no longer in control of device */
5272 	ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
5273 	ctrl_ext &= ~E1000_CTRL_EXT_DRV_LOAD;
5274 	E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
5275 }
5276 
5277 /*
5278  * igb_atomic_reserve - Atomic decrease operation
5279  */
5280 int
5281 igb_atomic_reserve(uint32_t *count_p, uint32_t n)
5282 {
5283 	uint32_t oldval;
5284 	uint32_t newval;
5285 
5286 	/* ATOMICALLY */
5287 	do {
5288 		oldval = *count_p;
5289 		if (oldval < n)
5290 			return (-1);
5291 		newval = oldval - n;
5292 	} while (atomic_cas_32(count_p, oldval, newval) != oldval);
5293 
5294 	return (newval);
5295 }
5296 
5297 /*
5298  * FMA support
5299  */
5300 
5301 int
5302 igb_check_acc_handle(ddi_acc_handle_t handle)
5303 {
5304 	ddi_fm_error_t de;
5305 
5306 	ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
5307 	ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
5308 	return (de.fme_status);
5309 }
5310 
5311 int
5312 igb_check_dma_handle(ddi_dma_handle_t handle)
5313 {
5314 	ddi_fm_error_t de;
5315 
5316 	ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
5317 	return (de.fme_status);
5318 }
5319 
5320 /*
5321  * The IO fault service error handling callback function
5322  */
5323 /*ARGSUSED*/
5324 static int
5325 igb_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
5326 {
5327 	/*
5328 	 * as the driver can always deal with an error in any dma or
5329 	 * access handle, we can just return the fme_status value.
5330 	 */
5331 	pci_ereport_post(dip, err, NULL);
5332 	return (err->fme_status);
5333 }
5334 
5335 static void
5336 igb_fm_init(igb_t *igb)
5337 {
5338 	ddi_iblock_cookie_t iblk;
5339 	int fma_dma_flag;
5340 
5341 	/* Only register with IO Fault Services if we have some capability */
5342 	if (igb->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
5343 		igb_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
5344 	} else {
5345 		igb_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
5346 	}
5347 
5348 	if (igb->fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
5349 		fma_dma_flag = 1;
5350 	} else {
5351 		fma_dma_flag = 0;
5352 	}
5353 
5354 	(void) igb_set_fma_flags(fma_dma_flag);
5355 
5356 	if (igb->fm_capabilities) {
5357 
5358 		/* Register capabilities with IO Fault Services */
5359 		ddi_fm_init(igb->dip, &igb->fm_capabilities, &iblk);
5360 
5361 		/*
5362 		 * Initialize pci ereport capabilities if ereport capable
5363 		 */
5364 		if (DDI_FM_EREPORT_CAP(igb->fm_capabilities) ||
5365 		    DDI_FM_ERRCB_CAP(igb->fm_capabilities))
5366 			pci_ereport_setup(igb->dip);
5367 
5368 		/*
5369 		 * Register error callback if error callback capable
5370 		 */
5371 		if (DDI_FM_ERRCB_CAP(igb->fm_capabilities))
5372 			ddi_fm_handler_register(igb->dip,
5373 			    igb_fm_error_cb, (void*) igb);
5374 	}
5375 }
5376 
5377 static void
5378 igb_fm_fini(igb_t *igb)
5379 {
5380 	/* Only unregister FMA capabilities if we registered some */
5381 	if (igb->fm_capabilities) {
5382 
5383 		/*
5384 		 * Release any resources allocated by pci_ereport_setup()
5385 		 */
5386 		if (DDI_FM_EREPORT_CAP(igb->fm_capabilities) ||
5387 		    DDI_FM_ERRCB_CAP(igb->fm_capabilities))
5388 			pci_ereport_teardown(igb->dip);
5389 
5390 		/*
5391 		 * Un-register error callback if error callback capable
5392 		 */
5393 		if (DDI_FM_ERRCB_CAP(igb->fm_capabilities))
5394 			ddi_fm_handler_unregister(igb->dip);
5395 
5396 		/* Unregister from IO Fault Services */
5397 		ddi_fm_fini(igb->dip);
5398 	}
5399 }
5400 
5401 void
5402 igb_fm_ereport(igb_t *igb, char *detail)
5403 {
5404 	uint64_t ena;
5405 	char buf[FM_MAX_CLASS];
5406 
5407 	(void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
5408 	ena = fm_ena_generate(0, FM_ENA_FMT1);
5409 	if (DDI_FM_EREPORT_CAP(igb->fm_capabilities)) {
5410 		ddi_fm_ereport_post(igb->dip, buf, ena, DDI_NOSLEEP,
5411 		    FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
5412 	}
5413 }
5414 
5415 static int
5416 igb_ufm_fill_image(ddi_ufm_handle_t *ufmh, void *arg, uint_t imgno,
5417     ddi_ufm_image_t *imgp)
5418 {
5419 	igb_t *igb = arg;
5420 	const char *type;
5421 
5422 	if (imgno != 0) {
5423 		return (EINVAL);
5424 	}
5425 
5426 	ddi_ufm_image_set_desc(imgp, "NVM");
5427 	ddi_ufm_image_set_nslots(imgp, 1);
5428 	switch (igb->hw.nvm.type) {
5429 	case e1000_nvm_eeprom_spi:
5430 		type = "SPI EEPROM";
5431 		break;
5432 	case e1000_nvm_eeprom_microwire:
5433 		type = "Microwire EEPROM";
5434 		break;
5435 	case e1000_nvm_invm:
5436 		type = "Internal NVM";
5437 		break;
5438 	case e1000_nvm_flash_hw:
5439 	case e1000_nvm_flash_sw:
5440 		type = "Flash";
5441 		break;
5442 	default:
5443 		type = NULL;
5444 		break;
5445 	}
5446 
5447 	if (type != NULL) {
5448 		nvlist_t *nvl;
5449 
5450 		nvl = fnvlist_alloc();
5451 		fnvlist_add_string(nvl, "image-type", type);
5452 		/*
5453 		 * The DDI takes ownership of the nvlist_t at this point.
5454 		 */
5455 		ddi_ufm_image_set_misc(imgp, nvl);
5456 	}
5457 
5458 	return (0);
5459 }
5460 
5461 static int
5462 igb_ufm_fill_slot(ddi_ufm_handle_t *ufmh, void *arg, uint_t imgno,
5463     uint_t slotno, ddi_ufm_slot_t *slotp)
5464 {
5465 	igb_t *igb = arg;
5466 	char *ver;
5467 
5468 	if (imgno != 0 || slotno != 0) {
5469 		return (EINVAL);
5470 	}
5471 
5472 	if (ddi_prop_lookup_string(DDI_DEV_T_ANY, igb->dip, DDI_PROP_DONTPASS,
5473 	    "nvm-version", &ver) == 0) {
5474 		ddi_ufm_slot_set_version(slotp, ver);
5475 		ddi_prop_free(ver);
5476 	} else {
5477 		ddi_ufm_slot_set_version(slotp, "unknown");
5478 	}
5479 
5480 	ddi_ufm_slot_set_attrs(slotp, DDI_UFM_ATTR_ACTIVE |
5481 	    DDI_UFM_ATTR_READABLE | DDI_UFM_ATTR_WRITEABLE);
5482 	ddi_ufm_slot_set_imgsize(slotp, igb->hw.nvm.word_size * 2);
5483 	return (0);
5484 }
5485 
5486 static int
5487 igb_ufm_getcaps(ddi_ufm_handle_t *ufmh, void *arg, ddi_ufm_cap_t *caps)
5488 {
5489 	igb_t *igb = arg;
5490 
5491 	*caps = 0;
5492 	if (igb->hw.nvm.type != e1000_nvm_none &&
5493 	    igb->hw.nvm.type != e1000_nvm_unknown) {
5494 		*caps |= DDI_UFM_CAP_REPORT;
5495 
5496 		if (igb->hw.nvm.ops.read != NULL) {
5497 			*caps |= DDI_UFM_CAP_READIMG;
5498 		}
5499 	}
5500 
5501 	return (0);
5502 }
5503 
5504 static int
5505 igb_ufm_readimg(ddi_ufm_handle_t *ufmh, void *arg, uint_t imgno, uint_t slotno,
5506     uint64_t len, uint64_t offset, void *buf, uint64_t *nread)
5507 {
5508 	igb_t *igb = arg;
5509 	uint16_t wordoff, nwords, *buf16 = buf;
5510 	uint32_t imgsize = igb->hw.nvm.word_size * 2;
5511 	int ret;
5512 
5513 	if (imgno != 0 || slotno != 0) {
5514 		return (EINVAL);
5515 	}
5516 
5517 	if (len > imgsize || offset > imgsize || len + offset > imgsize) {
5518 		return (EINVAL);
5519 	}
5520 
5521 	if (igb->hw.nvm.ops.read == NULL) {
5522 		return (ENOTSUP);
5523 	}
5524 
5525 	/*
5526 	 * Hardware provides us a means to read 16-bit words. For the time
5527 	 * being, restrict offset and length to be 2 byte aligned. We should
5528 	 * probably reduce this restriction. We could probably just use a bounce
5529 	 * buffer.
5530 	 */
5531 	if ((offset % 2) != 0 || (len % 2) != 0) {
5532 		return (EINVAL);
5533 	}
5534 
5535 	wordoff = offset >> 1;
5536 	nwords = len >> 1;
5537 	mutex_enter(&igb->gen_lock);
5538 	ret = e1000_read_nvm(&igb->hw, wordoff, nwords, buf16);
5539 	mutex_exit(&igb->gen_lock);
5540 
5541 	if (ret == 0) {
5542 		uint16_t i;
5543 		*nread = len;
5544 		for (i = 0; i < nwords; i++) {
5545 			buf16[i] = LE_16(buf16[i]);
5546 		}
5547 	} else {
5548 		ret = EIO;
5549 	}
5550 
5551 	return (ret);
5552 }
5553