xref: /titanic_44/usr/src/uts/common/io/ixgbe/ixgbe_main.c (revision ca35ed50b0189df73b8ea3eb509c9d582312df7c)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright(c) 2007-2008 Intel Corporation. All rights reserved.
24  */
25 
26 /*
27  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
28  * Use is subject to license terms.
29  */
30 
31 
32 #include "ixgbe_sw.h"
33 
34 static char ident[] = "Intel 10Gb Ethernet";
35 
36 /*
37  * Local function protoypes
38  */
39 static int ixgbe_register_mac(ixgbe_t *);
40 static int ixgbe_identify_hardware(ixgbe_t *);
41 static int ixgbe_regs_map(ixgbe_t *);
42 static void ixgbe_init_properties(ixgbe_t *);
43 static int ixgbe_init_driver_settings(ixgbe_t *);
44 static void ixgbe_init_locks(ixgbe_t *);
45 static void ixgbe_destroy_locks(ixgbe_t *);
46 static int ixgbe_init(ixgbe_t *);
47 static int ixgbe_chip_start(ixgbe_t *);
48 static void ixgbe_chip_stop(ixgbe_t *);
49 static int ixgbe_reset(ixgbe_t *);
50 static void ixgbe_tx_clean(ixgbe_t *);
51 static boolean_t ixgbe_tx_drain(ixgbe_t *);
52 static boolean_t ixgbe_rx_drain(ixgbe_t *);
53 static int ixgbe_alloc_rings(ixgbe_t *);
54 static int ixgbe_init_rings(ixgbe_t *);
55 static void ixgbe_free_rings(ixgbe_t *);
56 static void ixgbe_fini_rings(ixgbe_t *);
57 static void ixgbe_setup_rings(ixgbe_t *);
58 static void ixgbe_setup_rx(ixgbe_t *);
59 static void ixgbe_setup_tx(ixgbe_t *);
60 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *);
61 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *);
62 static void ixgbe_setup_rss(ixgbe_t *);
63 static void ixgbe_init_unicst(ixgbe_t *);
64 static int ixgbe_unicst_set(ixgbe_t *, const uint8_t *, int);
65 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *);
66 static void ixgbe_setup_multicst(ixgbe_t *);
67 static void ixgbe_get_hw_state(ixgbe_t *);
68 static void ixgbe_get_conf(ixgbe_t *);
69 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int);
70 static boolean_t ixgbe_driver_link_check(ixgbe_t *);
71 static void ixgbe_local_timer(void *);
72 static void ixgbe_arm_watchdog_timer(ixgbe_t *);
73 static void ixgbe_start_watchdog_timer(ixgbe_t *);
74 static void ixgbe_restart_watchdog_timer(ixgbe_t *);
75 static void ixgbe_stop_watchdog_timer(ixgbe_t *);
76 static void ixgbe_disable_adapter_interrupts(ixgbe_t *);
77 static void ixgbe_enable_adapter_interrupts(ixgbe_t *);
78 static boolean_t is_valid_mac_addr(uint8_t *);
79 static boolean_t ixgbe_stall_check(ixgbe_t *);
80 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t);
81 static void ixgbe_set_internal_mac_loopback(ixgbe_t *);
82 static boolean_t ixgbe_find_mac_address(ixgbe_t *);
83 static int ixgbe_alloc_intrs(ixgbe_t *);
84 static int ixgbe_alloc_intr_handles(ixgbe_t *, int);
85 static int ixgbe_add_intr_handlers(ixgbe_t *);
86 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int);
87 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int);
88 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t);
89 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t);
90 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t);
91 static int ixgbe_map_rings_to_vectors(ixgbe_t *);
92 static void ixgbe_setup_adapter_vector(ixgbe_t *);
93 static void ixgbe_rem_intr_handlers(ixgbe_t *);
94 static void ixgbe_rem_intrs(ixgbe_t *);
95 static int ixgbe_enable_intrs(ixgbe_t *);
96 static int ixgbe_disable_intrs(ixgbe_t *);
97 static uint_t ixgbe_intr_legacy(void *, void *);
98 static uint_t ixgbe_intr_msi(void *, void *);
99 static uint_t ixgbe_intr_rx_tx(void *, void *);
100 static uint_t ixgbe_intr_other(void *, void *);
101 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *);
102 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *);
103 static void ixgbe_intr_other_work(ixgbe_t *);
104 static void ixgbe_get_driver_control(struct ixgbe_hw *);
105 static int ixgbe_addmac(void *, const uint8_t *);
106 static int ixgbe_remmac(void *, const uint8_t *);
107 static void ixgbe_release_driver_control(struct ixgbe_hw *);
108 
109 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t);
110 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t);
111 static int ixgbe_resume(dev_info_t *);
112 static int ixgbe_suspend(dev_info_t *);
113 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *);
114 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *);
115 
116 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err,
117     const void *impl_data);
118 static void ixgbe_fm_init(ixgbe_t *);
119 static void ixgbe_fm_fini(ixgbe_t *);
120 
121 static struct cb_ops ixgbe_cb_ops = {
122 	nulldev,		/* cb_open */
123 	nulldev,		/* cb_close */
124 	nodev,			/* cb_strategy */
125 	nodev,			/* cb_print */
126 	nodev,			/* cb_dump */
127 	nodev,			/* cb_read */
128 	nodev,			/* cb_write */
129 	nodev,			/* cb_ioctl */
130 	nodev,			/* cb_devmap */
131 	nodev,			/* cb_mmap */
132 	nodev,			/* cb_segmap */
133 	nochpoll,		/* cb_chpoll */
134 	ddi_prop_op,		/* cb_prop_op */
135 	NULL,			/* cb_stream */
136 	D_MP | D_HOTPLUG,	/* cb_flag */
137 	CB_REV,			/* cb_rev */
138 	nodev,			/* cb_aread */
139 	nodev			/* cb_awrite */
140 };
141 
142 static struct dev_ops ixgbe_dev_ops = {
143 	DEVO_REV,		/* devo_rev */
144 	0,			/* devo_refcnt */
145 	NULL,			/* devo_getinfo */
146 	nulldev,		/* devo_identify */
147 	nulldev,		/* devo_probe */
148 	ixgbe_attach,		/* devo_attach */
149 	ixgbe_detach,		/* devo_detach */
150 	nodev,			/* devo_reset */
151 	&ixgbe_cb_ops,		/* devo_cb_ops */
152 	NULL,			/* devo_bus_ops */
153 	ddi_power,		/* devo_power */
154 	ddi_quiesce_not_supported,	/* devo_quiesce */
155 };
156 
157 static struct modldrv ixgbe_modldrv = {
158 	&mod_driverops,		/* Type of module.  This one is a driver */
159 	ident,			/* Discription string */
160 	&ixgbe_dev_ops		/* driver ops */
161 };
162 
163 static struct modlinkage ixgbe_modlinkage = {
164 	MODREV_1, &ixgbe_modldrv, NULL
165 };
166 
167 /*
168  * Access attributes for register mapping
169  */
170 ddi_device_acc_attr_t ixgbe_regs_acc_attr = {
171 	DDI_DEVICE_ATTR_V0,
172 	DDI_STRUCTURE_LE_ACC,
173 	DDI_STRICTORDER_ACC,
174 	DDI_FLAGERR_ACC
175 };
176 
177 /*
178  * Loopback property
179  */
180 static lb_property_t lb_normal = {
181 	normal,	"normal", IXGBE_LB_NONE
182 };
183 
184 static lb_property_t lb_mac = {
185 	internal, "MAC", IXGBE_LB_INTERNAL_MAC
186 };
187 
188 #define	IXGBE_M_CALLBACK_FLAGS	(MC_IOCTL | MC_GETCAPAB)
189 
190 static mac_callbacks_t ixgbe_m_callbacks = {
191 	IXGBE_M_CALLBACK_FLAGS,
192 	ixgbe_m_stat,
193 	ixgbe_m_start,
194 	ixgbe_m_stop,
195 	ixgbe_m_promisc,
196 	ixgbe_m_multicst,
197 	NULL,
198 	NULL,
199 	ixgbe_m_ioctl,
200 	ixgbe_m_getcapab
201 };
202 
203 /*
204  * Module Initialization Functions.
205  */
206 
207 int
208 _init(void)
209 {
210 	int status;
211 
212 	mac_init_ops(&ixgbe_dev_ops, MODULE_NAME);
213 
214 	status = mod_install(&ixgbe_modlinkage);
215 
216 	if (status != DDI_SUCCESS) {
217 		mac_fini_ops(&ixgbe_dev_ops);
218 	}
219 
220 	return (status);
221 }
222 
223 int
224 _fini(void)
225 {
226 	int status;
227 
228 	status = mod_remove(&ixgbe_modlinkage);
229 
230 	if (status == DDI_SUCCESS) {
231 		mac_fini_ops(&ixgbe_dev_ops);
232 	}
233 
234 	return (status);
235 }
236 
237 int
238 _info(struct modinfo *modinfop)
239 {
240 	int status;
241 
242 	status = mod_info(&ixgbe_modlinkage, modinfop);
243 
244 	return (status);
245 }
246 
247 /*
248  * ixgbe_attach - Driver attach.
249  *
250  * This function is the device specific initialization entry
251  * point. This entry point is required and must be written.
252  * The DDI_ATTACH command must be provided in the attach entry
253  * point. When attach() is called with cmd set to DDI_ATTACH,
254  * all normal kernel services (such as kmem_alloc(9F)) are
255  * available for use by the driver.
256  *
257  * The attach() function will be called once for each instance
258  * of  the  device  on  the  system with cmd set to DDI_ATTACH.
259  * Until attach() succeeds, the only driver entry points which
260  * may be called are open(9E) and getinfo(9E).
261  */
262 static int
263 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
264 {
265 	ixgbe_t *ixgbe;
266 	struct ixgbe_osdep *osdep;
267 	struct ixgbe_hw *hw;
268 	int instance;
269 
270 	/*
271 	 * Check the command and perform corresponding operations
272 	 */
273 	switch (cmd) {
274 	default:
275 		return (DDI_FAILURE);
276 
277 	case DDI_RESUME:
278 		return (ixgbe_resume(devinfo));
279 
280 	case DDI_ATTACH:
281 		break;
282 	}
283 
284 	/* Get the device instance */
285 	instance = ddi_get_instance(devinfo);
286 
287 	/* Allocate memory for the instance data structure */
288 	ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP);
289 
290 	ixgbe->dip = devinfo;
291 	ixgbe->instance = instance;
292 
293 	hw = &ixgbe->hw;
294 	osdep = &ixgbe->osdep;
295 	hw->back = osdep;
296 	osdep->ixgbe = ixgbe;
297 
298 	/* Attach the instance pointer to the dev_info data structure */
299 	ddi_set_driver_private(devinfo, ixgbe);
300 
301 	/*
302 	 * Initialize for fma support
303 	 */
304 	ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE,
305 	    0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
306 	    DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
307 	ixgbe_fm_init(ixgbe);
308 	ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT;
309 
310 	/*
311 	 * Map PCI config space registers
312 	 */
313 	if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
314 		ixgbe_error(ixgbe, "Failed to map PCI configurations");
315 		goto attach_fail;
316 	}
317 	ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
318 
319 	/*
320 	 * Identify the chipset family
321 	 */
322 	if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) {
323 		ixgbe_error(ixgbe, "Failed to identify hardware");
324 		goto attach_fail;
325 	}
326 
327 	/*
328 	 * Map device registers
329 	 */
330 	if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) {
331 		ixgbe_error(ixgbe, "Failed to map device registers");
332 		goto attach_fail;
333 	}
334 	ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP;
335 
336 	/*
337 	 * Initialize driver parameters
338 	 */
339 	ixgbe_init_properties(ixgbe);
340 	ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS;
341 
342 	/*
343 	 * Allocate interrupts
344 	 */
345 	if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) {
346 		ixgbe_error(ixgbe, "Failed to allocate interrupts");
347 		goto attach_fail;
348 	}
349 	ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
350 
351 	/*
352 	 * Allocate rx/tx rings based on the ring numbers.
353 	 * The actual numbers of rx/tx rings are decided by the number of
354 	 * allocated interrupt vectors, so we should allocate the rings after
355 	 * interrupts are allocated.
356 	 */
357 	if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) {
358 		ixgbe_error(ixgbe, "Failed to allocate rx and tx rings");
359 		goto attach_fail;
360 	}
361 	ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS;
362 
363 	/*
364 	 * Map rings to interrupt vectors
365 	 */
366 	if (ixgbe_map_rings_to_vectors(ixgbe) != IXGBE_SUCCESS) {
367 		ixgbe_error(ixgbe, "Failed to map rings to vectors");
368 		goto attach_fail;
369 	}
370 
371 	/*
372 	 * Add interrupt handlers
373 	 */
374 	if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
375 		ixgbe_error(ixgbe, "Failed to add interrupt handlers");
376 		goto attach_fail;
377 	}
378 	ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
379 
380 	/*
381 	 * Initialize driver parameters
382 	 */
383 	if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) {
384 		ixgbe_error(ixgbe, "Failed to initialize driver settings");
385 		goto attach_fail;
386 	}
387 
388 	/*
389 	 * Initialize mutexes for this device.
390 	 * Do this before enabling the interrupt handler and
391 	 * register the softint to avoid the condition where
392 	 * interrupt handler can try using uninitialized mutex.
393 	 */
394 	ixgbe_init_locks(ixgbe);
395 	ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS;
396 
397 	/*
398 	 * Initialize chipset hardware
399 	 */
400 	if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) {
401 		ixgbe_error(ixgbe, "Failed to initialize adapter");
402 		goto attach_fail;
403 	}
404 	ixgbe->attach_progress |= ATTACH_PROGRESS_INIT;
405 
406 	if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) {
407 		ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
408 		goto attach_fail;
409 	}
410 
411 	/*
412 	 * Initialize DMA and hardware settings for rx/tx rings
413 	 */
414 	if (ixgbe_init_rings(ixgbe) != IXGBE_SUCCESS) {
415 		ixgbe_error(ixgbe, "Failed to initialize rings");
416 		goto attach_fail;
417 	}
418 	ixgbe->attach_progress |= ATTACH_PROGRESS_INIT_RINGS;
419 
420 	/*
421 	 * Initialize statistics
422 	 */
423 	if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) {
424 		ixgbe_error(ixgbe, "Failed to initialize statistics");
425 		goto attach_fail;
426 	}
427 	ixgbe->attach_progress |= ATTACH_PROGRESS_STATS;
428 
429 	/*
430 	 * Initialize NDD parameters
431 	 */
432 	if (ixgbe_nd_init(ixgbe) != IXGBE_SUCCESS) {
433 		ixgbe_error(ixgbe, "Failed to initialize ndd");
434 		goto attach_fail;
435 	}
436 	ixgbe->attach_progress |= ATTACH_PROGRESS_NDD;
437 
438 	/*
439 	 * Register the driver to the MAC
440 	 */
441 	if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) {
442 		ixgbe_error(ixgbe, "Failed to register MAC");
443 		goto attach_fail;
444 	}
445 	ixgbe->attach_progress |= ATTACH_PROGRESS_MAC;
446 
447 	/*
448 	 * Now that mutex locks are initialized, and the chip is also
449 	 * initialized, enable interrupts.
450 	 */
451 	if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
452 		ixgbe_error(ixgbe, "Failed to enable DDI interrupts");
453 		goto attach_fail;
454 	}
455 	ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
456 
457 	ixgbe->ixgbe_state |= IXGBE_INITIALIZED;
458 
459 	return (DDI_SUCCESS);
460 
461 attach_fail:
462 	ixgbe_unconfigure(devinfo, ixgbe);
463 	return (DDI_FAILURE);
464 }
465 
466 /*
467  * ixgbe_detach - Driver detach.
468  *
469  * The detach() function is the complement of the attach routine.
470  * If cmd is set to DDI_DETACH, detach() is used to remove  the
471  * state  associated  with  a  given  instance of a device node
472  * prior to the removal of that instance from the system.
473  *
474  * The detach() function will be called once for each  instance
475  * of the device for which there has been a successful attach()
476  * once there are no longer  any  opens  on  the  device.
477  *
478  * Interrupts routine are disabled, All memory allocated by this
479  * driver are freed.
480  */
481 static int
482 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
483 {
484 	ixgbe_t *ixgbe;
485 
486 	/*
487 	 * Check detach command
488 	 */
489 	switch (cmd) {
490 	default:
491 		return (DDI_FAILURE);
492 
493 	case DDI_SUSPEND:
494 		return (ixgbe_suspend(devinfo));
495 
496 	case DDI_DETACH:
497 		break;
498 	}
499 
500 
501 	/*
502 	 * Get the pointer to the driver private data structure
503 	 */
504 	ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
505 	if (ixgbe == NULL)
506 		return (DDI_FAILURE);
507 
508 	/*
509 	 * Unregister MAC. If failed, we have to fail the detach
510 	 */
511 	if (mac_unregister(ixgbe->mac_hdl) != 0) {
512 		ixgbe_error(ixgbe, "Failed to unregister MAC");
513 		return (DDI_FAILURE);
514 	}
515 	ixgbe->attach_progress &= ~ATTACH_PROGRESS_MAC;
516 
517 	/*
518 	 * If the device is still running, it needs to be stopped first.
519 	 * This check is necessary because under some specific circumstances,
520 	 * the detach routine can be called without stopping the interface
521 	 * first.
522 	 */
523 	mutex_enter(&ixgbe->gen_lock);
524 	if (ixgbe->ixgbe_state & IXGBE_STARTED) {
525 		ixgbe->ixgbe_state &= ~IXGBE_STARTED;
526 		ixgbe_stop(ixgbe);
527 		mutex_exit(&ixgbe->gen_lock);
528 		/* Disable and stop the watchdog timer */
529 		ixgbe_disable_watchdog_timer(ixgbe);
530 	} else
531 		mutex_exit(&ixgbe->gen_lock);
532 
533 	/*
534 	 * Check if there are still rx buffers held by the upper layer.
535 	 * If so, fail the detach.
536 	 */
537 	if (!ixgbe_rx_drain(ixgbe))
538 		return (DDI_FAILURE);
539 
540 	/*
541 	 * Do the remaining unconfigure routines
542 	 */
543 	ixgbe_unconfigure(devinfo, ixgbe);
544 
545 	return (DDI_SUCCESS);
546 }
547 
548 static void
549 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe)
550 {
551 	/*
552 	 * Disable interrupt
553 	 */
554 	if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
555 		(void) ixgbe_disable_intrs(ixgbe);
556 	}
557 
558 	/*
559 	 * Unregister MAC
560 	 */
561 	if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) {
562 		(void) mac_unregister(ixgbe->mac_hdl);
563 	}
564 
565 	/*
566 	 * Free ndd parameters
567 	 */
568 	if (ixgbe->attach_progress & ATTACH_PROGRESS_NDD) {
569 		ixgbe_nd_cleanup(ixgbe);
570 	}
571 
572 	/*
573 	 * Free statistics
574 	 */
575 	if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) {
576 		kstat_delete((kstat_t *)ixgbe->ixgbe_ks);
577 	}
578 
579 	/*
580 	 * Remove interrupt handlers
581 	 */
582 	if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
583 		ixgbe_rem_intr_handlers(ixgbe);
584 	}
585 
586 	/*
587 	 * Remove interrupts
588 	 */
589 	if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
590 		ixgbe_rem_intrs(ixgbe);
591 	}
592 
593 	/*
594 	 * Remove driver properties
595 	 */
596 	if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) {
597 		(void) ddi_prop_remove_all(devinfo);
598 	}
599 
600 	/*
601 	 * Release the DMA resources of rx/tx rings
602 	 */
603 	if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT_RINGS) {
604 		ixgbe_fini_rings(ixgbe);
605 	}
606 
607 	/*
608 	 * Stop the chipset
609 	 */
610 	if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) {
611 		mutex_enter(&ixgbe->gen_lock);
612 		ixgbe_chip_stop(ixgbe);
613 		mutex_exit(&ixgbe->gen_lock);
614 	}
615 
616 	/*
617 	 * Free register handle
618 	 */
619 	if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) {
620 		if (ixgbe->osdep.reg_handle != NULL)
621 			ddi_regs_map_free(&ixgbe->osdep.reg_handle);
622 	}
623 
624 	/*
625 	 * Free PCI config handle
626 	 */
627 	if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) {
628 		if (ixgbe->osdep.cfg_handle != NULL)
629 			pci_config_teardown(&ixgbe->osdep.cfg_handle);
630 	}
631 
632 	/*
633 	 * Free locks
634 	 */
635 	if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) {
636 		ixgbe_destroy_locks(ixgbe);
637 	}
638 
639 	/*
640 	 * Free the rx/tx rings
641 	 */
642 	if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) {
643 		ixgbe_free_rings(ixgbe);
644 	}
645 
646 	/*
647 	 * Unregister FMA capabilities
648 	 */
649 	if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) {
650 		ixgbe_fm_fini(ixgbe);
651 	}
652 
653 	/*
654 	 * Free the driver data structure
655 	 */
656 	kmem_free(ixgbe, sizeof (ixgbe_t));
657 
658 	ddi_set_driver_private(devinfo, NULL);
659 }
660 
661 /*
662  * ixgbe_register_mac - Register the driver and its function pointers with
663  * the GLD interface.
664  */
665 static int
666 ixgbe_register_mac(ixgbe_t *ixgbe)
667 {
668 	struct ixgbe_hw *hw = &ixgbe->hw;
669 	mac_register_t *mac;
670 	int status;
671 
672 	if ((mac = mac_alloc(MAC_VERSION)) == NULL)
673 		return (IXGBE_FAILURE);
674 
675 	mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
676 	mac->m_driver = ixgbe;
677 	mac->m_dip = ixgbe->dip;
678 	mac->m_src_addr = hw->mac.addr;
679 	mac->m_callbacks = &ixgbe_m_callbacks;
680 	mac->m_min_sdu = 0;
681 	mac->m_max_sdu = ixgbe->default_mtu;
682 	mac->m_margin = VLAN_TAGSZ;
683 	mac->m_v12n = MAC_VIRT_LEVEL1;
684 
685 	status = mac_register(mac, &ixgbe->mac_hdl);
686 
687 	mac_free(mac);
688 
689 	return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE);
690 }
691 
692 /*
693  * ixgbe_identify_hardware - Identify the type of the chipset.
694  */
695 static int
696 ixgbe_identify_hardware(ixgbe_t *ixgbe)
697 {
698 	struct ixgbe_hw *hw = &ixgbe->hw;
699 	struct ixgbe_osdep *osdep = &ixgbe->osdep;
700 
701 	/*
702 	 * Get the device id
703 	 */
704 	hw->vendor_id =
705 	    pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID);
706 	hw->device_id =
707 	    pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID);
708 	hw->revision_id =
709 	    pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID);
710 	hw->subsystem_device_id =
711 	    pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID);
712 	hw->subsystem_vendor_id =
713 	    pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID);
714 
715 	return (IXGBE_SUCCESS);
716 }
717 
718 /*
719  * ixgbe_regs_map - Map the device registers.
720  *
721  */
722 static int
723 ixgbe_regs_map(ixgbe_t *ixgbe)
724 {
725 	dev_info_t *devinfo = ixgbe->dip;
726 	struct ixgbe_hw *hw = &ixgbe->hw;
727 	struct ixgbe_osdep *osdep = &ixgbe->osdep;
728 	off_t mem_size;
729 
730 	/*
731 	 * First get the size of device registers to be mapped.
732 	 */
733 	if (ddi_dev_regsize(devinfo, 1, &mem_size) != DDI_SUCCESS) {
734 		return (IXGBE_FAILURE);
735 	}
736 
737 	/*
738 	 * Call ddi_regs_map_setup() to map registers
739 	 */
740 	if ((ddi_regs_map_setup(devinfo, 1,
741 	    (caddr_t *)&hw->hw_addr, 0,
742 	    mem_size, &ixgbe_regs_acc_attr,
743 	    &osdep->reg_handle)) != DDI_SUCCESS) {
744 		return (IXGBE_FAILURE);
745 	}
746 
747 	return (IXGBE_SUCCESS);
748 }
749 
750 /*
751  * ixgbe_init_properties - Initialize driver properties.
752  */
753 static void
754 ixgbe_init_properties(ixgbe_t *ixgbe)
755 {
756 	/*
757 	 * Get conf file properties, including link settings
758 	 * jumbo frames, ring number, descriptor number, etc.
759 	 */
760 	ixgbe_get_conf(ixgbe);
761 }
762 
763 /*
764  * ixgbe_init_driver_settings - Initialize driver settings.
765  *
766  * The settings include hardware function pointers, bus information,
767  * rx/tx rings settings, link state, and any other parameters that
768  * need to be setup during driver initialization.
769  */
770 static int
771 ixgbe_init_driver_settings(ixgbe_t *ixgbe)
772 {
773 	struct ixgbe_hw *hw = &ixgbe->hw;
774 	dev_info_t *devinfo = ixgbe->dip;
775 	ixgbe_rx_ring_t *rx_ring;
776 	ixgbe_tx_ring_t *tx_ring;
777 	uint32_t rx_size;
778 	uint32_t tx_size;
779 	int i;
780 
781 	/*
782 	 * Initialize chipset specific hardware function pointers
783 	 */
784 	if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) {
785 		return (IXGBE_FAILURE);
786 	}
787 
788 	/*
789 	 * Get the system page size
790 	 */
791 	ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1);
792 
793 	/*
794 	 * Set rx buffer size
795 	 *
796 	 * The IP header alignment room is counted in the calculation.
797 	 * The rx buffer size is in unit of 1K that is required by the
798 	 * chipset hardware.
799 	 */
800 	rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM;
801 	ixgbe->rx_buf_size = ((rx_size >> 10) +
802 	    ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
803 
804 	/*
805 	 * Set tx buffer size
806 	 */
807 	tx_size = ixgbe->max_frame_size;
808 	ixgbe->tx_buf_size = ((tx_size >> 10) +
809 	    ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
810 
811 	/*
812 	 * Initialize rx/tx rings parameters
813 	 */
814 	for (i = 0; i < ixgbe->num_rx_rings; i++) {
815 		rx_ring = &ixgbe->rx_rings[i];
816 		rx_ring->index = i;
817 		rx_ring->ixgbe = ixgbe;
818 
819 		rx_ring->ring_size = ixgbe->rx_ring_size;
820 		rx_ring->free_list_size = ixgbe->rx_ring_size;
821 		rx_ring->copy_thresh = ixgbe->rx_copy_thresh;
822 		rx_ring->limit_per_intr = ixgbe->rx_limit_per_intr;
823 	}
824 
825 	for (i = 0; i < ixgbe->num_tx_rings; i++) {
826 		tx_ring = &ixgbe->tx_rings[i];
827 		tx_ring->index = i;
828 		tx_ring->ixgbe = ixgbe;
829 		if (ixgbe->tx_head_wb_enable)
830 			tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb;
831 		else
832 			tx_ring->tx_recycle = ixgbe_tx_recycle_legacy;
833 
834 		tx_ring->ring_size = ixgbe->tx_ring_size;
835 		tx_ring->free_list_size = ixgbe->tx_ring_size +
836 		    (ixgbe->tx_ring_size >> 1);
837 		tx_ring->copy_thresh = ixgbe->tx_copy_thresh;
838 		tx_ring->recycle_thresh = ixgbe->tx_recycle_thresh;
839 		tx_ring->overload_thresh = ixgbe->tx_overload_thresh;
840 	tx_ring->resched_thresh = ixgbe->tx_resched_thresh;
841 	}
842 
843 	/*
844 	 * Initialize values of interrupt throttling rate
845 	 */
846 	for (i = 1; i < IXGBE_MAX_RING_VECTOR; i++)
847 		ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0];
848 
849 	/*
850 	 * The initial link state should be "unknown"
851 	 */
852 	ixgbe->link_state = LINK_STATE_UNKNOWN;
853 	return (IXGBE_SUCCESS);
854 }
855 
856 /*
857  * ixgbe_init_locks - Initialize locks.
858  */
859 static void
860 ixgbe_init_locks(ixgbe_t *ixgbe)
861 {
862 	ixgbe_rx_ring_t *rx_ring;
863 	ixgbe_tx_ring_t *tx_ring;
864 	int i;
865 
866 	for (i = 0; i < ixgbe->num_rx_rings; i++) {
867 		rx_ring = &ixgbe->rx_rings[i];
868 		mutex_init(&rx_ring->rx_lock, NULL,
869 		    MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
870 		mutex_init(&rx_ring->recycle_lock, NULL,
871 		    MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
872 	}
873 
874 	for (i = 0; i < ixgbe->num_tx_rings; i++) {
875 		tx_ring = &ixgbe->tx_rings[i];
876 		mutex_init(&tx_ring->tx_lock, NULL,
877 		    MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
878 		mutex_init(&tx_ring->recycle_lock, NULL,
879 		    MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
880 		mutex_init(&tx_ring->tcb_head_lock, NULL,
881 		    MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
882 		mutex_init(&tx_ring->tcb_tail_lock, NULL,
883 		    MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
884 	}
885 
886 	mutex_init(&ixgbe->gen_lock, NULL,
887 	    MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
888 
889 	mutex_init(&ixgbe->watchdog_lock, NULL,
890 	    MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
891 }
892 
893 /*
894  * ixgbe_destroy_locks - Destroy locks.
895  */
896 static void
897 ixgbe_destroy_locks(ixgbe_t *ixgbe)
898 {
899 	ixgbe_rx_ring_t *rx_ring;
900 	ixgbe_tx_ring_t *tx_ring;
901 	int i;
902 
903 	for (i = 0; i < ixgbe->num_rx_rings; i++) {
904 		rx_ring = &ixgbe->rx_rings[i];
905 		mutex_destroy(&rx_ring->rx_lock);
906 		mutex_destroy(&rx_ring->recycle_lock);
907 	}
908 
909 	for (i = 0; i < ixgbe->num_tx_rings; i++) {
910 		tx_ring = &ixgbe->tx_rings[i];
911 		mutex_destroy(&tx_ring->tx_lock);
912 		mutex_destroy(&tx_ring->recycle_lock);
913 		mutex_destroy(&tx_ring->tcb_head_lock);
914 		mutex_destroy(&tx_ring->tcb_tail_lock);
915 	}
916 
917 	mutex_destroy(&ixgbe->gen_lock);
918 	mutex_destroy(&ixgbe->watchdog_lock);
919 }
920 
921 static int
922 ixgbe_resume(dev_info_t *devinfo)
923 {
924 	ixgbe_t *ixgbe;
925 
926 	ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
927 	if (ixgbe == NULL)
928 		return (DDI_FAILURE);
929 
930 	mutex_enter(&ixgbe->gen_lock);
931 
932 	if (ixgbe->ixgbe_state & IXGBE_STARTED) {
933 		if (ixgbe_start(ixgbe) != IXGBE_SUCCESS) {
934 			mutex_exit(&ixgbe->gen_lock);
935 			return (DDI_FAILURE);
936 		}
937 
938 		/*
939 		 * Enable and start the watchdog timer
940 		 */
941 		ixgbe_enable_watchdog_timer(ixgbe);
942 	}
943 
944 	ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED;
945 
946 	mutex_exit(&ixgbe->gen_lock);
947 
948 	return (DDI_SUCCESS);
949 }
950 
951 static int
952 ixgbe_suspend(dev_info_t *devinfo)
953 {
954 	ixgbe_t *ixgbe;
955 
956 	ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
957 	if (ixgbe == NULL)
958 		return (DDI_FAILURE);
959 
960 	mutex_enter(&ixgbe->gen_lock);
961 
962 	ixgbe->ixgbe_state |= IXGBE_SUSPENDED;
963 
964 	ixgbe_stop(ixgbe);
965 
966 	mutex_exit(&ixgbe->gen_lock);
967 
968 	/*
969 	 * Disable and stop the watchdog timer
970 	 */
971 	ixgbe_disable_watchdog_timer(ixgbe);
972 
973 	return (DDI_SUCCESS);
974 }
975 
976 /*
977  * ixgbe_init - Initialize the device.
978  */
979 static int
980 ixgbe_init(ixgbe_t *ixgbe)
981 {
982 	struct ixgbe_hw *hw = &ixgbe->hw;
983 
984 	mutex_enter(&ixgbe->gen_lock);
985 
986 	/*
987 	 * Reset chipset to put the hardware in a known state
988 	 * before we try to do anything with the eeprom.
989 	 */
990 	if (ixgbe_reset_hw(hw) != IXGBE_SUCCESS) {
991 		ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
992 		goto init_fail;
993 	}
994 
995 	/*
996 	 * Need to init eeprom before validating the checksum.
997 	 */
998 	if (ixgbe_init_eeprom_params(hw) < 0) {
999 		ixgbe_error(ixgbe,
1000 		    "Unable to intitialize the eeprom interface.");
1001 		ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1002 		goto init_fail;
1003 	}
1004 
1005 	/*
1006 	 * NVM validation
1007 	 */
1008 	if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1009 		/*
1010 		 * Some PCI-E parts fail the first check due to
1011 		 * the link being in sleep state.  Call it again,
1012 		 * if it fails a second time it's a real issue.
1013 		 */
1014 		if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1015 			ixgbe_error(ixgbe,
1016 			    "Invalid NVM checksum. Please contact "
1017 			    "the vendor to update the NVM.");
1018 			ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1019 			goto init_fail;
1020 		}
1021 	}
1022 
1023 	/*
1024 	 * Setup default flow control thresholds - enable/disable
1025 	 * & flow control type is controlled by ixgbe.conf
1026 	 */
1027 	hw->fc.high_water = DEFAULT_FCRTH;
1028 	hw->fc.low_water = DEFAULT_FCRTL;
1029 	hw->fc.pause_time = DEFAULT_FCPAUSE;
1030 	hw->fc.send_xon = B_TRUE;
1031 
1032 	/*
1033 	 * Don't wait for auto-negotiation to complete
1034 	 */
1035 	hw->phy.autoneg_wait_to_complete = B_FALSE;
1036 
1037 	/*
1038 	 * Initialize link settings
1039 	 */
1040 	(void) ixgbe_driver_setup_link(ixgbe, B_FALSE);
1041 
1042 	/*
1043 	 * Initialize the chipset hardware
1044 	 */
1045 	if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1046 		ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1047 		goto init_fail;
1048 	}
1049 
1050 	if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) {
1051 		goto init_fail;
1052 	}
1053 	if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1054 		goto init_fail;
1055 	}
1056 
1057 	mutex_exit(&ixgbe->gen_lock);
1058 	return (IXGBE_SUCCESS);
1059 
1060 init_fail:
1061 	/*
1062 	 * Reset PHY
1063 	 */
1064 	(void) ixgbe_reset_phy(hw);
1065 
1066 	mutex_exit(&ixgbe->gen_lock);
1067 	ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1068 	return (IXGBE_FAILURE);
1069 }
1070 
1071 /*
1072  * ixgbe_init_rings - Allocate DMA resources for all rx/tx rings and
1073  * initialize relevant hardware settings.
1074  */
1075 static int
1076 ixgbe_init_rings(ixgbe_t *ixgbe)
1077 {
1078 	int i;
1079 
1080 	/*
1081 	 * Allocate buffers for all the rx/tx rings
1082 	 */
1083 	if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS)
1084 		return (IXGBE_FAILURE);
1085 
1086 	/*
1087 	 * Setup the rx/tx rings
1088 	 */
1089 	mutex_enter(&ixgbe->gen_lock);
1090 
1091 	for (i = 0; i < ixgbe->num_rx_rings; i++)
1092 		mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1093 	for (i = 0; i < ixgbe->num_tx_rings; i++)
1094 		mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1095 
1096 	ixgbe_setup_rings(ixgbe);
1097 
1098 	for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1099 		mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1100 	for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1101 		mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1102 
1103 	mutex_exit(&ixgbe->gen_lock);
1104 
1105 	return (IXGBE_SUCCESS);
1106 }
1107 
1108 /*
1109  * ixgbe_fini_rings - Release DMA resources of all rx/tx rings.
1110  */
1111 static void
1112 ixgbe_fini_rings(ixgbe_t *ixgbe)
1113 {
1114 	/*
1115 	 * Release the DMA/memory resources of rx/tx rings
1116 	 */
1117 	ixgbe_free_dma(ixgbe);
1118 }
1119 
1120 /*
1121  * ixgbe_chip_start - Initialize and start the chipset hardware.
1122  */
1123 static int
1124 ixgbe_chip_start(ixgbe_t *ixgbe)
1125 {
1126 	struct ixgbe_hw *hw = &ixgbe->hw;
1127 	int i;
1128 
1129 	ASSERT(mutex_owned(&ixgbe->gen_lock));
1130 
1131 	/*
1132 	 * Get the mac address
1133 	 * This function should handle SPARC case correctly.
1134 	 */
1135 	if (!ixgbe_find_mac_address(ixgbe)) {
1136 		ixgbe_error(ixgbe, "Failed to get the mac address");
1137 		return (IXGBE_FAILURE);
1138 	}
1139 
1140 	/*
1141 	 * Validate the mac address
1142 	 */
1143 	(void) ixgbe_init_rx_addrs(hw);
1144 	if (!is_valid_mac_addr(hw->mac.addr)) {
1145 		ixgbe_error(ixgbe, "Invalid mac address");
1146 		return (IXGBE_FAILURE);
1147 	}
1148 
1149 	/*
1150 	 * Configure/Initialize hardware
1151 	 */
1152 	if (ixgbe_init_hw(hw) != IXGBE_SUCCESS) {
1153 		ixgbe_error(ixgbe, "Failed to initialize hardware");
1154 		return (IXGBE_FAILURE);
1155 	}
1156 
1157 	/*
1158 	 * Setup adapter interrupt vectors
1159 	 */
1160 	ixgbe_setup_adapter_vector(ixgbe);
1161 
1162 	/*
1163 	 * Initialize unicast addresses.
1164 	 */
1165 	ixgbe_init_unicst(ixgbe);
1166 
1167 	/*
1168 	 * Setup and initialize the mctable structures.
1169 	 */
1170 	ixgbe_setup_multicst(ixgbe);
1171 
1172 	/*
1173 	 * Set interrupt throttling rate
1174 	 */
1175 	for (i = 0; i < ixgbe->intr_cnt; i++)
1176 		IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]);
1177 
1178 	/*
1179 	 * Save the state of the phy
1180 	 */
1181 	ixgbe_get_hw_state(ixgbe);
1182 
1183 	/*
1184 	 * Make sure driver has control
1185 	 */
1186 	ixgbe_get_driver_control(hw);
1187 
1188 	return (IXGBE_SUCCESS);
1189 }
1190 
1191 /*
1192  * ixgbe_chip_stop - Stop the chipset hardware
1193  */
1194 static void
1195 ixgbe_chip_stop(ixgbe_t *ixgbe)
1196 {
1197 	struct ixgbe_hw *hw = &ixgbe->hw;
1198 
1199 	ASSERT(mutex_owned(&ixgbe->gen_lock));
1200 
1201 	/*
1202 	 * Tell firmware driver is no longer in control
1203 	 */
1204 	ixgbe_release_driver_control(hw);
1205 
1206 	/*
1207 	 * Reset the chipset
1208 	 */
1209 	(void) ixgbe_reset_hw(hw);
1210 
1211 	/*
1212 	 * Reset PHY
1213 	 */
1214 	(void) ixgbe_reset_phy(hw);
1215 }
1216 
1217 /*
1218  * ixgbe_reset - Reset the chipset and re-start the driver.
1219  *
1220  * It involves stopping and re-starting the chipset,
1221  * and re-configuring the rx/tx rings.
1222  */
1223 static int
1224 ixgbe_reset(ixgbe_t *ixgbe)
1225 {
1226 	int i;
1227 
1228 	mutex_enter(&ixgbe->gen_lock);
1229 
1230 	ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED);
1231 	ixgbe->ixgbe_state &= ~IXGBE_STARTED;
1232 
1233 	/*
1234 	 * Disable the adapter interrupts to stop any rx/tx activities
1235 	 * before draining pending data and resetting hardware.
1236 	 */
1237 	ixgbe_disable_adapter_interrupts(ixgbe);
1238 
1239 	/*
1240 	 * Drain the pending transmit packets
1241 	 */
1242 	(void) ixgbe_tx_drain(ixgbe);
1243 
1244 	for (i = 0; i < ixgbe->num_rx_rings; i++)
1245 		mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1246 	for (i = 0; i < ixgbe->num_tx_rings; i++)
1247 		mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1248 
1249 	/*
1250 	 * Stop the chipset hardware
1251 	 */
1252 	ixgbe_chip_stop(ixgbe);
1253 
1254 	/*
1255 	 * Clean the pending tx data/resources
1256 	 */
1257 	ixgbe_tx_clean(ixgbe);
1258 
1259 	/*
1260 	 * Start the chipset hardware
1261 	 */
1262 	if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1263 		ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1264 		goto reset_failure;
1265 	}
1266 
1267 	if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1268 		goto reset_failure;
1269 	}
1270 
1271 	/*
1272 	 * Setup the rx/tx rings
1273 	 */
1274 	ixgbe_setup_rings(ixgbe);
1275 
1276 	/*
1277 	 * Enable adapter interrupts
1278 	 * The interrupts must be enabled after the driver state is START
1279 	 */
1280 	ixgbe_enable_adapter_interrupts(ixgbe);
1281 
1282 	for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1283 		mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1284 	for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1285 		mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1286 
1287 	ixgbe->ixgbe_state |= IXGBE_STARTED;
1288 	mutex_exit(&ixgbe->gen_lock);
1289 
1290 	return (IXGBE_SUCCESS);
1291 
1292 reset_failure:
1293 	for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1294 		mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1295 	for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1296 		mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1297 
1298 	mutex_exit(&ixgbe->gen_lock);
1299 
1300 	ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1301 
1302 	return (IXGBE_FAILURE);
1303 }
1304 
1305 /*
1306  * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources.
1307  */
1308 static void
1309 ixgbe_tx_clean(ixgbe_t *ixgbe)
1310 {
1311 	ixgbe_tx_ring_t *tx_ring;
1312 	tx_control_block_t *tcb;
1313 	link_list_t pending_list;
1314 	uint32_t desc_num;
1315 	struct ixgbe_hw *hw = &ixgbe->hw;
1316 	int i, j;
1317 
1318 	LINK_LIST_INIT(&pending_list);
1319 
1320 	for (i = 0; i < ixgbe->num_tx_rings; i++) {
1321 		tx_ring = &ixgbe->tx_rings[i];
1322 
1323 		mutex_enter(&tx_ring->recycle_lock);
1324 
1325 		/*
1326 		 * Clean the pending tx data - the pending packets in the
1327 		 * work_list that have no chances to be transmitted again.
1328 		 *
1329 		 * We must ensure the chipset is stopped or the link is down
1330 		 * before cleaning the transmit packets.
1331 		 */
1332 		desc_num = 0;
1333 		for (j = 0; j < tx_ring->ring_size; j++) {
1334 			tcb = tx_ring->work_list[j];
1335 			if (tcb != NULL) {
1336 				desc_num += tcb->desc_num;
1337 
1338 				tx_ring->work_list[j] = NULL;
1339 
1340 				ixgbe_free_tcb(tcb);
1341 
1342 				LIST_PUSH_TAIL(&pending_list, &tcb->link);
1343 			}
1344 		}
1345 
1346 		if (desc_num > 0) {
1347 			atomic_add_32(&tx_ring->tbd_free, desc_num);
1348 			ASSERT(tx_ring->tbd_free == tx_ring->ring_size);
1349 
1350 			/*
1351 			 * Reset the head and tail pointers of the tbd ring;
1352 			 * Reset the writeback head if it's enable.
1353 			 */
1354 			tx_ring->tbd_head = 0;
1355 			tx_ring->tbd_tail = 0;
1356 			if (ixgbe->tx_head_wb_enable)
1357 				*tx_ring->tbd_head_wb = 0;
1358 
1359 			IXGBE_WRITE_REG(&ixgbe->hw,
1360 			    IXGBE_TDH(tx_ring->index), 0);
1361 			IXGBE_WRITE_REG(&ixgbe->hw,
1362 			    IXGBE_TDT(tx_ring->index), 0);
1363 		}
1364 
1365 		mutex_exit(&tx_ring->recycle_lock);
1366 
1367 		/*
1368 		 * Add the tx control blocks in the pending list to
1369 		 * the free list.
1370 		 */
1371 		ixgbe_put_free_list(tx_ring, &pending_list);
1372 	}
1373 }
1374 
1375 /*
1376  * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be
1377  * transmitted.
1378  */
1379 static boolean_t
1380 ixgbe_tx_drain(ixgbe_t *ixgbe)
1381 {
1382 	ixgbe_tx_ring_t *tx_ring;
1383 	boolean_t done;
1384 	int i, j;
1385 
1386 	/*
1387 	 * Wait for a specific time to allow pending tx packets
1388 	 * to be transmitted.
1389 	 *
1390 	 * Check the counter tbd_free to see if transmission is done.
1391 	 * No lock protection is needed here.
1392 	 *
1393 	 * Return B_TRUE if all pending packets have been transmitted;
1394 	 * Otherwise return B_FALSE;
1395 	 */
1396 	for (i = 0; i < TX_DRAIN_TIME; i++) {
1397 
1398 		done = B_TRUE;
1399 		for (j = 0; j < ixgbe->num_tx_rings; j++) {
1400 			tx_ring = &ixgbe->tx_rings[j];
1401 			done = done &&
1402 			    (tx_ring->tbd_free == tx_ring->ring_size);
1403 		}
1404 
1405 		if (done)
1406 			break;
1407 
1408 		msec_delay(1);
1409 	}
1410 
1411 	return (done);
1412 }
1413 
1414 /*
1415  * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer.
1416  */
1417 static boolean_t
1418 ixgbe_rx_drain(ixgbe_t *ixgbe)
1419 {
1420 	ixgbe_rx_ring_t *rx_ring;
1421 	boolean_t done;
1422 	int i, j;
1423 
1424 	/*
1425 	 * Polling the rx free list to check if those rx buffers held by
1426 	 * the upper layer are released.
1427 	 *
1428 	 * Check the counter rcb_free to see if all pending buffers are
1429 	 * released. No lock protection is needed here.
1430 	 *
1431 	 * Return B_TRUE if all pending buffers have been released;
1432 	 * Otherwise return B_FALSE;
1433 	 */
1434 	for (i = 0; i < RX_DRAIN_TIME; i++) {
1435 
1436 		done = B_TRUE;
1437 		for (j = 0; j < ixgbe->num_rx_rings; j++) {
1438 			rx_ring = &ixgbe->rx_rings[j];
1439 			done = done &&
1440 			    (rx_ring->rcb_free == rx_ring->free_list_size);
1441 		}
1442 
1443 		if (done)
1444 			break;
1445 
1446 		msec_delay(1);
1447 	}
1448 
1449 	return (done);
1450 }
1451 
1452 /*
1453  * ixgbe_start - Start the driver/chipset.
1454  */
1455 int
1456 ixgbe_start(ixgbe_t *ixgbe)
1457 {
1458 	int i;
1459 
1460 	ASSERT(mutex_owned(&ixgbe->gen_lock));
1461 
1462 	for (i = 0; i < ixgbe->num_rx_rings; i++)
1463 		mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1464 	for (i = 0; i < ixgbe->num_tx_rings; i++)
1465 		mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1466 
1467 	/*
1468 	 * Start the chipset hardware
1469 	 */
1470 	if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1471 		ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1472 		goto start_failure;
1473 	}
1474 
1475 	if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1476 		goto start_failure;
1477 	}
1478 
1479 	/*
1480 	 * Setup the rx/tx rings
1481 	 */
1482 	ixgbe_setup_rings(ixgbe);
1483 
1484 	/*
1485 	 * Enable adapter interrupts
1486 	 * The interrupts must be enabled after the driver state is START
1487 	 */
1488 	ixgbe_enable_adapter_interrupts(ixgbe);
1489 
1490 	for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1491 		mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1492 	for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1493 		mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1494 
1495 	return (IXGBE_SUCCESS);
1496 
1497 start_failure:
1498 	for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1499 		mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1500 	for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1501 		mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1502 
1503 	ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1504 
1505 	return (IXGBE_FAILURE);
1506 }
1507 
1508 /*
1509  * ixgbe_stop - Stop the driver/chipset.
1510  */
1511 void
1512 ixgbe_stop(ixgbe_t *ixgbe)
1513 {
1514 	int i;
1515 
1516 	ASSERT(mutex_owned(&ixgbe->gen_lock));
1517 
1518 	/*
1519 	 * Disable the adapter interrupts
1520 	 */
1521 	ixgbe_disable_adapter_interrupts(ixgbe);
1522 
1523 	/*
1524 	 * Drain the pending tx packets
1525 	 */
1526 	(void) ixgbe_tx_drain(ixgbe);
1527 
1528 	for (i = 0; i < ixgbe->num_rx_rings; i++)
1529 		mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1530 	for (i = 0; i < ixgbe->num_tx_rings; i++)
1531 		mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1532 
1533 	/*
1534 	 * Stop the chipset hardware
1535 	 */
1536 	ixgbe_chip_stop(ixgbe);
1537 
1538 	if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1539 		ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1540 	}
1541 
1542 	/*
1543 	 * Clean the pending tx data/resources
1544 	 */
1545 	ixgbe_tx_clean(ixgbe);
1546 
1547 	for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1548 		mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1549 	for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1550 		mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1551 }
1552 
1553 /*
1554  * ixgbe_alloc_rings - Allocate memory space for rx/tx rings.
1555  */
1556 static int
1557 ixgbe_alloc_rings(ixgbe_t *ixgbe)
1558 {
1559 	/*
1560 	 * Allocate memory space for rx rings
1561 	 */
1562 	ixgbe->rx_rings = kmem_zalloc(
1563 	    sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings,
1564 	    KM_NOSLEEP);
1565 
1566 	if (ixgbe->rx_rings == NULL) {
1567 		return (IXGBE_FAILURE);
1568 	}
1569 
1570 	/*
1571 	 * Allocate memory space for tx rings
1572 	 */
1573 	ixgbe->tx_rings = kmem_zalloc(
1574 	    sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings,
1575 	    KM_NOSLEEP);
1576 
1577 	if (ixgbe->tx_rings == NULL) {
1578 		kmem_free(ixgbe->rx_rings,
1579 		    sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
1580 		ixgbe->rx_rings = NULL;
1581 		return (IXGBE_FAILURE);
1582 	}
1583 
1584 	/*
1585 	 * Allocate memory space for rx ring groups
1586 	 */
1587 	ixgbe->rx_groups = kmem_zalloc(
1588 	    sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups,
1589 	    KM_NOSLEEP);
1590 
1591 	if (ixgbe->rx_groups == NULL) {
1592 		kmem_free(ixgbe->rx_rings,
1593 		    sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
1594 		kmem_free(ixgbe->tx_rings,
1595 		    sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
1596 		ixgbe->rx_rings = NULL;
1597 		ixgbe->tx_rings = NULL;
1598 		return (IXGBE_FAILURE);
1599 	}
1600 
1601 	return (IXGBE_SUCCESS);
1602 }
1603 
1604 /*
1605  * ixgbe_free_rings - Free the memory space of rx/tx rings.
1606  */
1607 static void
1608 ixgbe_free_rings(ixgbe_t *ixgbe)
1609 {
1610 	if (ixgbe->rx_rings != NULL) {
1611 		kmem_free(ixgbe->rx_rings,
1612 		    sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
1613 		ixgbe->rx_rings = NULL;
1614 	}
1615 
1616 	if (ixgbe->tx_rings != NULL) {
1617 		kmem_free(ixgbe->tx_rings,
1618 		    sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
1619 		ixgbe->tx_rings = NULL;
1620 	}
1621 
1622 	if (ixgbe->rx_groups != NULL) {
1623 		kmem_free(ixgbe->rx_groups,
1624 		    sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups);
1625 		ixgbe->rx_groups = NULL;
1626 	}
1627 }
1628 
1629 /*
1630  * ixgbe_setup_rings - Setup rx/tx rings.
1631  */
1632 static void
1633 ixgbe_setup_rings(ixgbe_t *ixgbe)
1634 {
1635 	/*
1636 	 * Setup the rx/tx rings, including the following:
1637 	 *
1638 	 * 1. Setup the descriptor ring and the control block buffers;
1639 	 * 2. Initialize necessary registers for receive/transmit;
1640 	 * 3. Initialize software pointers/parameters for receive/transmit;
1641 	 */
1642 	ixgbe_setup_rx(ixgbe);
1643 
1644 	ixgbe_setup_tx(ixgbe);
1645 }
1646 
1647 static void
1648 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring)
1649 {
1650 	ixgbe_t *ixgbe = rx_ring->ixgbe;
1651 	struct ixgbe_hw *hw = &ixgbe->hw;
1652 	rx_control_block_t *rcb;
1653 	union ixgbe_adv_rx_desc	*rbd;
1654 	uint32_t size;
1655 	uint32_t buf_low;
1656 	uint32_t buf_high;
1657 	uint32_t reg_val;
1658 	int i;
1659 
1660 	ASSERT(mutex_owned(&rx_ring->rx_lock));
1661 	ASSERT(mutex_owned(&ixgbe->gen_lock));
1662 
1663 	for (i = 0; i < ixgbe->rx_ring_size; i++) {
1664 		rcb = rx_ring->work_list[i];
1665 		rbd = &rx_ring->rbd_ring[i];
1666 
1667 		rbd->read.pkt_addr = rcb->rx_buf.dma_address;
1668 		rbd->read.hdr_addr = NULL;
1669 	}
1670 
1671 	/*
1672 	 * Initialize the length register
1673 	 */
1674 	size = rx_ring->ring_size * sizeof (union ixgbe_adv_rx_desc);
1675 	IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->index), size);
1676 
1677 	/*
1678 	 * Initialize the base address registers
1679 	 */
1680 	buf_low = (uint32_t)rx_ring->rbd_area.dma_address;
1681 	buf_high = (uint32_t)(rx_ring->rbd_area.dma_address >> 32);
1682 	IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->index), buf_high);
1683 	IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->index), buf_low);
1684 
1685 	/*
1686 	 * Setup head & tail pointers
1687 	 */
1688 	IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->index), rx_ring->ring_size - 1);
1689 	IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->index), 0);
1690 
1691 	rx_ring->rbd_next = 0;
1692 
1693 	/*
1694 	 * Note: Considering the case that the chipset is being reset
1695 	 * and there are still some buffers held by the upper layer,
1696 	 * we should not reset the values of rcb_head, rcb_tail and
1697 	 * rcb_free if the state is not IXGBE_UNKNOWN.
1698 	 */
1699 	if (ixgbe->ixgbe_state == IXGBE_UNKNOWN) {
1700 		rx_ring->rcb_head = 0;
1701 		rx_ring->rcb_tail = 0;
1702 		rx_ring->rcb_free = rx_ring->free_list_size;
1703 	}
1704 
1705 	/*
1706 	 * Setup the Receive Descriptor Control Register (RXDCTL)
1707 	 * PTHRESH=32 descriptors (half the internal cache)
1708 	 * HTHRESH=0 descriptors (to minimize latency on fetch)
1709 	 * WTHRESH defaults to 1 (writeback each descriptor)
1710 	 */
1711 	reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->index));
1712 	reg_val |= IXGBE_RXDCTL_ENABLE;	/* enable queue */
1713 	reg_val |= 0x0020;		/* pthresh */
1714 	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->index), reg_val);
1715 
1716 	/*
1717 	 * Setup the Split and Replication Receive Control Register.
1718 	 * Set the rx buffer size and the advanced descriptor type.
1719 	 */
1720 	reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) |
1721 	    IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
1722 
1723 	IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->index), reg_val);
1724 }
1725 
1726 static void
1727 ixgbe_setup_rx(ixgbe_t *ixgbe)
1728 {
1729 	ixgbe_rx_ring_t *rx_ring;
1730 	struct ixgbe_hw *hw = &ixgbe->hw;
1731 	ixgbe_rx_group_t *rx_group;
1732 	uint32_t reg_val;
1733 	uint32_t ring_mapping;
1734 	int i;
1735 
1736 	/*
1737 	 * Set filter control in FCTRL to accept broadcast packets and do
1738 	 * not pass pause frames to host.  Flow control settings are already
1739 	 * in this register, so preserve them.
1740 	 */
1741 	reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1742 	reg_val |= IXGBE_FCTRL_BAM;	/* broadcast accept mode */
1743 	reg_val |= IXGBE_FCTRL_DPF;	/* discard pause frames */
1744 	IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val);
1745 
1746 	/*
1747 	 * Enable the receive unit.  This must be done after filter
1748 	 * control is set in FCTRL.
1749 	 */
1750 	reg_val = (IXGBE_RXCTRL_RXEN	/* Enable Receive Unit */
1751 	    | IXGBE_RXCTRL_DMBYPS);	/* descriptor monitor bypass */
1752 	IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, reg_val);
1753 
1754 	/*
1755 	 * ixgbe_setup_rx_ring must be called after configuring RXCTRL
1756 	 */
1757 	for (i = 0; i < ixgbe->num_rx_rings; i++) {
1758 		rx_ring = &ixgbe->rx_rings[i];
1759 		ixgbe_setup_rx_ring(rx_ring);
1760 	}
1761 
1762 	/*
1763 	 * Setup rx groups.
1764 	 */
1765 	for (i = 0; i < ixgbe->num_rx_groups; i++) {
1766 		rx_group = &ixgbe->rx_groups[i];
1767 		rx_group->index = i;
1768 		rx_group->ixgbe = ixgbe;
1769 	}
1770 
1771 	/*
1772 	 * Setup the per-ring statistics mapping.
1773 	 */
1774 	ring_mapping = 0;
1775 	for (i = 0; i < ixgbe->num_rx_rings; i++) {
1776 		ring_mapping |= (i & 0xF) << (8 * (i & 0x3));
1777 		if ((i & 0x3) == 0x3) {
1778 			IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i >> 2), ring_mapping);
1779 			ring_mapping = 0;
1780 		}
1781 	}
1782 	if ((i & 0x3) != 0x3)
1783 		IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i >> 2), ring_mapping);
1784 
1785 	/*
1786 	 * The Max Frame Size in MHADD will be internally increased by four
1787 	 * bytes if the packet has a VLAN field, so includes MTU, ethernet
1788 	 * header and frame check sequence.
1789 	 */
1790 	reg_val = (ixgbe->default_mtu + sizeof (struct ether_header)
1791 	    + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT;
1792 	IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val);
1793 
1794 	/*
1795 	 * Setup Jumbo Frame enable bit
1796 	 */
1797 	if (ixgbe->default_mtu > ETHERMTU) {
1798 		reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
1799 		reg_val |= IXGBE_HLREG0_JUMBOEN;
1800 		IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
1801 	}
1802 
1803 	/*
1804 	 * Hardware checksum settings
1805 	 */
1806 	if (ixgbe->rx_hcksum_enable) {
1807 		reg_val = IXGBE_RXCSUM_IPPCSE;	/* IP checksum */
1808 		IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val);
1809 	}
1810 
1811 	/*
1812 	 * Setup RSS for multiple receive queues
1813 	 */
1814 	if (ixgbe->num_rx_rings > 1)
1815 		ixgbe_setup_rss(ixgbe);
1816 }
1817 
1818 static void
1819 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring)
1820 {
1821 	ixgbe_t *ixgbe = tx_ring->ixgbe;
1822 	struct ixgbe_hw *hw = &ixgbe->hw;
1823 	uint32_t size;
1824 	uint32_t buf_low;
1825 	uint32_t buf_high;
1826 	uint32_t reg_val;
1827 
1828 	ASSERT(mutex_owned(&tx_ring->tx_lock));
1829 	ASSERT(mutex_owned(&ixgbe->gen_lock));
1830 
1831 	/*
1832 	 * Initialize the length register
1833 	 */
1834 	size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc);
1835 	IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size);
1836 
1837 	/*
1838 	 * Initialize the base address registers
1839 	 */
1840 	buf_low = (uint32_t)tx_ring->tbd_area.dma_address;
1841 	buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32);
1842 	IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low);
1843 	IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high);
1844 
1845 	/*
1846 	 * setup TXDCTL(tx_ring->index)
1847 	 */
1848 	reg_val = IXGBE_TXDCTL_ENABLE;
1849 	IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val);
1850 
1851 	/*
1852 	 * Setup head & tail pointers
1853 	 */
1854 	IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0);
1855 	IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0);
1856 
1857 	/*
1858 	 * Setup head write-back
1859 	 */
1860 	if (ixgbe->tx_head_wb_enable) {
1861 		/*
1862 		 * The memory of the head write-back is allocated using
1863 		 * the extra tbd beyond the tail of the tbd ring.
1864 		 */
1865 		tx_ring->tbd_head_wb = (uint32_t *)
1866 		    ((uintptr_t)tx_ring->tbd_area.address + size);
1867 		*tx_ring->tbd_head_wb = 0;
1868 
1869 		buf_low = (uint32_t)
1870 		    (tx_ring->tbd_area.dma_address + size);
1871 		buf_high = (uint32_t)
1872 		    ((tx_ring->tbd_area.dma_address + size) >> 32);
1873 
1874 		/* Set the head write-back enable bit */
1875 		buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE;
1876 
1877 		IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low);
1878 		IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high);
1879 
1880 		/*
1881 		 * Turn off relaxed ordering for head write back or it will
1882 		 * cause problems with the tx recycling
1883 		 */
1884 		reg_val = IXGBE_READ_REG(hw,
1885 		    IXGBE_DCA_TXCTRL(tx_ring->index));
1886 		reg_val &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
1887 		IXGBE_WRITE_REG(hw,
1888 		    IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
1889 	} else {
1890 		tx_ring->tbd_head_wb = NULL;
1891 	}
1892 
1893 	tx_ring->tbd_head = 0;
1894 	tx_ring->tbd_tail = 0;
1895 	tx_ring->tbd_free = tx_ring->ring_size;
1896 
1897 	/*
1898 	 * Note: Considering the case that the chipset is being reset,
1899 	 * and there are still some tcb in the pending list,
1900 	 * we should not reset the values of tcb_head, tcb_tail and
1901 	 * tcb_free if the state is not IXGBE_UNKNOWN.
1902 	 */
1903 	if (ixgbe->ixgbe_state == IXGBE_UNKNOWN) {
1904 		tx_ring->tcb_head = 0;
1905 		tx_ring->tcb_tail = 0;
1906 		tx_ring->tcb_free = tx_ring->free_list_size;
1907 	}
1908 
1909 	/*
1910 	 * Initialize the s/w context structure
1911 	 */
1912 	bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t));
1913 }
1914 
1915 static void
1916 ixgbe_setup_tx(ixgbe_t *ixgbe)
1917 {
1918 	struct ixgbe_hw *hw = &ixgbe->hw;
1919 	ixgbe_tx_ring_t *tx_ring;
1920 	uint32_t reg_val;
1921 	uint32_t ring_mapping;
1922 	int i;
1923 
1924 	for (i = 0; i < ixgbe->num_tx_rings; i++) {
1925 		tx_ring = &ixgbe->tx_rings[i];
1926 		ixgbe_setup_tx_ring(tx_ring);
1927 	}
1928 
1929 	/*
1930 	 * Setup the per-ring statistics mapping.
1931 	 */
1932 	ring_mapping = 0;
1933 	for (i = 0; i < ixgbe->num_tx_rings; i++) {
1934 		ring_mapping |= (i & 0xF) << (8 * (i & 0x3));
1935 		if ((i & 0x3) == 0x3) {
1936 			IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping);
1937 			ring_mapping = 0;
1938 		}
1939 	}
1940 	if ((i & 0x3) != 0x3)
1941 		IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping);
1942 
1943 	/*
1944 	 * Enable CRC appending and TX padding (for short tx frames)
1945 	 */
1946 	reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
1947 	reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN;
1948 	IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
1949 }
1950 
1951 /*
1952  * ixgbe_setup_rss - Setup receive-side scaling feature.
1953  */
1954 static void
1955 ixgbe_setup_rss(ixgbe_t *ixgbe)
1956 {
1957 	struct ixgbe_hw *hw = &ixgbe->hw;
1958 	uint32_t i, mrqc, rxcsum;
1959 	uint32_t random;
1960 	uint32_t reta;
1961 
1962 	/*
1963 	 * Fill out redirection table
1964 	 */
1965 	reta = 0;
1966 	for (i = 0; i < 128; i++) {
1967 		reta = (reta << 8) | (i % ixgbe->num_rx_rings);
1968 		if ((i & 3) == 3)
1969 			IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
1970 	}
1971 
1972 	/*
1973 	 * Fill out hash function seeds with a random constant
1974 	 */
1975 	for (i = 0; i < 10; i++) {
1976 		(void) random_get_pseudo_bytes((uint8_t *)&random,
1977 		    sizeof (uint32_t));
1978 		IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
1979 	}
1980 
1981 	/*
1982 	 * Enable RSS & perform hash on these packet types
1983 	 */
1984 	mrqc = IXGBE_MRQC_RSSEN |
1985 	    IXGBE_MRQC_RSS_FIELD_IPV4 |
1986 	    IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
1987 	    IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
1988 	    IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
1989 	    IXGBE_MRQC_RSS_FIELD_IPV6_EX |
1990 	    IXGBE_MRQC_RSS_FIELD_IPV6 |
1991 	    IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
1992 	    IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
1993 	    IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
1994 	IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
1995 
1996 	/*
1997 	 * Disable Packet Checksum to enable RSS for multiple receive queues.
1998 	 * It is an adapter hardware limitation that Packet Checksum is
1999 	 * mutually exclusive with RSS.
2000 	 */
2001 	rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2002 	rxcsum |= IXGBE_RXCSUM_PCSD;
2003 	rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
2004 	IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2005 }
2006 
2007 /*
2008  * ixgbe_init_unicst - Initialize the unicast addresses.
2009  */
2010 static void
2011 ixgbe_init_unicst(ixgbe_t *ixgbe)
2012 {
2013 	struct ixgbe_hw *hw = &ixgbe->hw;
2014 	uint8_t *mac_addr;
2015 	int slot;
2016 	/*
2017 	 * Here we should consider two situations:
2018 	 *
2019 	 * 1. Chipset is initialized at the first time,
2020 	 *    Clear all the multiple unicast addresses.
2021 	 *
2022 	 * 2. Chipset is reset
2023 	 *    Recover the multiple unicast addresses from the
2024 	 *    software data structure to the RAR registers.
2025 	 */
2026 	if (!ixgbe->unicst_init) {
2027 		/*
2028 		 * Initialize the multiple unicast addresses
2029 		 */
2030 		ixgbe->unicst_total = MAX_NUM_UNICAST_ADDRESSES;
2031 		ixgbe->unicst_avail = ixgbe->unicst_total;
2032 		for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2033 			mac_addr = ixgbe->unicst_addr[slot].mac.addr;
2034 			bzero(mac_addr, ETHERADDRL);
2035 			(void) ixgbe_set_rar(hw, slot, mac_addr, NULL, NULL);
2036 			ixgbe->unicst_addr[slot].mac.set = 0;
2037 		}
2038 		ixgbe->unicst_init = B_TRUE;
2039 	} else {
2040 		/* Re-configure the RAR registers */
2041 		for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2042 			mac_addr = ixgbe->unicst_addr[slot].mac.addr;
2043 			if (ixgbe->unicst_addr[slot].mac.set == 1) {
2044 				(void) ixgbe_set_rar(hw, slot, mac_addr,
2045 				    NULL, IXGBE_RAH_AV);
2046 			} else {
2047 				bzero(mac_addr, ETHERADDRL);
2048 				(void) ixgbe_set_rar(hw, slot, mac_addr,
2049 				    NULL, NULL);
2050 			}
2051 		}
2052 	}
2053 }
2054 
2055 /*
2056  * ixgbe_unicst_set - Set the unicast address to the specified slot.
2057  */
2058 int
2059 ixgbe_unicst_set(ixgbe_t *ixgbe, const uint8_t *mac_addr,
2060     int slot)
2061 {
2062 	struct ixgbe_hw *hw = &ixgbe->hw;
2063 
2064 	ASSERT(mutex_owned(&ixgbe->gen_lock));
2065 
2066 	/*
2067 	 * Save the unicast address in the software data structure
2068 	 */
2069 	bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
2070 
2071 	/*
2072 	 * Set the unicast address to the RAR register
2073 	 */
2074 	(void) ixgbe_set_rar(hw, slot, (uint8_t *)mac_addr, NULL, IXGBE_RAH_AV);
2075 
2076 	if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
2077 		ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
2078 		return (EIO);
2079 	}
2080 
2081 	return (0);
2082 }
2083 
2084 /*
2085  * ixgbe_unicst_find - Find the slot for the specified unicast address
2086  */
2087 int
2088 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr)
2089 {
2090 	int slot;
2091 
2092 	ASSERT(mutex_owned(&ixgbe->gen_lock));
2093 
2094 	for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2095 		if (bcmp(ixgbe->unicst_addr[slot].mac.addr,
2096 		    mac_addr, ETHERADDRL) == 0)
2097 			return (slot);
2098 	}
2099 
2100 	return (-1);
2101 }
2102 
2103 /*
2104  * ixgbe_multicst_add - Add a multicst address.
2105  */
2106 int
2107 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr)
2108 {
2109 	ASSERT(mutex_owned(&ixgbe->gen_lock));
2110 
2111 	if ((multiaddr[0] & 01) == 0) {
2112 		return (EINVAL);
2113 	}
2114 
2115 	if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) {
2116 		return (ENOENT);
2117 	}
2118 
2119 	bcopy(multiaddr,
2120 	    &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL);
2121 	ixgbe->mcast_count++;
2122 
2123 	/*
2124 	 * Update the multicast table in the hardware
2125 	 */
2126 	ixgbe_setup_multicst(ixgbe);
2127 
2128 	if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
2129 		ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
2130 		return (EIO);
2131 	}
2132 
2133 	return (0);
2134 }
2135 
2136 /*
2137  * ixgbe_multicst_remove - Remove a multicst address.
2138  */
2139 int
2140 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr)
2141 {
2142 	int i;
2143 
2144 	ASSERT(mutex_owned(&ixgbe->gen_lock));
2145 
2146 	for (i = 0; i < ixgbe->mcast_count; i++) {
2147 		if (bcmp(multiaddr, &ixgbe->mcast_table[i],
2148 		    ETHERADDRL) == 0) {
2149 			for (i++; i < ixgbe->mcast_count; i++) {
2150 				ixgbe->mcast_table[i - 1] =
2151 				    ixgbe->mcast_table[i];
2152 			}
2153 			ixgbe->mcast_count--;
2154 			break;
2155 		}
2156 	}
2157 
2158 	/*
2159 	 * Update the multicast table in the hardware
2160 	 */
2161 	ixgbe_setup_multicst(ixgbe);
2162 
2163 	if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
2164 		ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
2165 		return (EIO);
2166 	}
2167 
2168 	return (0);
2169 }
2170 
2171 /*
2172  * ixgbe_setup_multicast - Setup multicast data structures.
2173  *
2174  * This routine initializes all of the multicast related structures
2175  * and save them in the hardware registers.
2176  */
2177 static void
2178 ixgbe_setup_multicst(ixgbe_t *ixgbe)
2179 {
2180 	uint8_t *mc_addr_list;
2181 	uint32_t mc_addr_count;
2182 	struct ixgbe_hw *hw = &ixgbe->hw;
2183 
2184 	ASSERT(mutex_owned(&ixgbe->gen_lock));
2185 
2186 	ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES);
2187 
2188 	mc_addr_list = (uint8_t *)ixgbe->mcast_table;
2189 	mc_addr_count = ixgbe->mcast_count;
2190 
2191 	/*
2192 	 * Update the multicast addresses to the MTA registers
2193 	 */
2194 	(void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count,
2195 	    ixgbe_mc_table_itr);
2196 }
2197 
2198 /*
2199  * ixgbe_get_conf - Get driver configurations set in driver.conf.
2200  *
2201  * This routine gets user-configured values out of the configuration
2202  * file ixgbe.conf.
2203  *
2204  * For each configurable value, there is a minimum, a maximum, and a
2205  * default.
2206  * If user does not configure a value, use the default.
2207  * If user configures below the minimum, use the minumum.
2208  * If user configures above the maximum, use the maxumum.
2209  */
2210 static void
2211 ixgbe_get_conf(ixgbe_t *ixgbe)
2212 {
2213 	struct ixgbe_hw *hw = &ixgbe->hw;
2214 	uint32_t flow_control;
2215 
2216 	/*
2217 	 * ixgbe driver supports the following user configurations:
2218 	 *
2219 	 * Jumbo frame configuration:
2220 	 *    default_mtu
2221 	 *
2222 	 * Ethernet flow control configuration:
2223 	 *    flow_control
2224 	 *
2225 	 * Multiple rings configurations:
2226 	 *    tx_queue_number
2227 	 *    tx_ring_size
2228 	 *    rx_queue_number
2229 	 *    rx_ring_size
2230 	 *
2231 	 * Call ixgbe_get_prop() to get the value for a specific
2232 	 * configuration parameter.
2233 	 */
2234 
2235 	/*
2236 	 * Jumbo frame configuration - max_frame_size controls host buffer
2237 	 * allocation, so includes MTU, ethernet header, vlan tag and
2238 	 * frame check sequence.
2239 	 */
2240 	ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU,
2241 	    MIN_MTU, MAX_MTU, DEFAULT_MTU);
2242 
2243 	ixgbe->max_frame_size = ixgbe->default_mtu +
2244 	    sizeof (struct ether_vlan_header) + ETHERFCSL;
2245 
2246 	/*
2247 	 * Ethernet flow control configuration
2248 	 */
2249 	flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL,
2250 	    ixgbe_fc_none, 3, ixgbe_fc_none);
2251 	if (flow_control == 3)
2252 		flow_control = ixgbe_fc_default;
2253 
2254 	hw->fc.type = flow_control;
2255 
2256 	/*
2257 	 * Multiple rings configurations
2258 	 */
2259 	ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM,
2260 	    MIN_TX_QUEUE_NUM, MAX_TX_QUEUE_NUM, DEFAULT_TX_QUEUE_NUM);
2261 	ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE,
2262 	    MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE);
2263 
2264 	ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM,
2265 	    MIN_RX_QUEUE_NUM, MAX_RX_QUEUE_NUM, DEFAULT_RX_QUEUE_NUM);
2266 	ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE,
2267 	    MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE);
2268 
2269 	/*
2270 	 * Multiple groups configuration
2271 	 */
2272 	ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM,
2273 	    MIN_RX_GROUP_NUM, MAX_RX_GROUP_NUM, DEFAULT_RX_GROUP_NUM);
2274 
2275 	ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE,
2276 	    0, 1, DEFAULT_MR_ENABLE);
2277 
2278 	if (ixgbe->mr_enable == B_FALSE) {
2279 		ixgbe->num_tx_rings = 1;
2280 		ixgbe->num_rx_rings = 1;
2281 		ixgbe->num_rx_groups = 1;
2282 	}
2283 
2284 	/*
2285 	 * Tunable used to force an interrupt type. The only use is
2286 	 * for testing of the lesser interrupt types.
2287 	 * 0 = don't force interrupt type
2288 	 * 1 = force interrupt type MSI-X
2289 	 * 2 = force interrupt type MSI
2290 	 * 3 = force interrupt type Legacy
2291 	 */
2292 	ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE,
2293 	    IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE);
2294 	ixgbe_log(ixgbe, "interrupt force: %d\n", ixgbe->intr_force);
2295 
2296 	ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE,
2297 	    0, 1, DEFAULT_TX_HCKSUM_ENABLE);
2298 	ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE,
2299 	    0, 1, DEFAULT_RX_HCKSUM_ENABLE);
2300 	ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE,
2301 	    0, 1, DEFAULT_LSO_ENABLE);
2302 	ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE,
2303 	    0, 1, DEFAULT_TX_HEAD_WB_ENABLE);
2304 
2305 	/*
2306 	 * ixgbe LSO needs the tx h/w checksum support.
2307 	 * LSO will be disabled if tx h/w checksum is not
2308 	 * enabled.
2309 	 */
2310 	if (ixgbe->tx_hcksum_enable == B_FALSE) {
2311 		ixgbe->lso_enable = B_FALSE;
2312 	}
2313 
2314 	ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD,
2315 	    MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
2316 	    DEFAULT_TX_COPY_THRESHOLD);
2317 	ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe,
2318 	    PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD,
2319 	    MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD);
2320 	ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe,
2321 	    PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD,
2322 	    MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD);
2323 	ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe,
2324 	    PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD,
2325 	    MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD);
2326 
2327 	ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD,
2328 	    MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
2329 	    DEFAULT_RX_COPY_THRESHOLD);
2330 	ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR,
2331 	    MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
2332 	    DEFAULT_RX_LIMIT_PER_INTR);
2333 
2334 	ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING,
2335 	    MIN_INTR_THROTTLING, MAX_INTR_THROTTLING,
2336 	    DEFAULT_INTR_THROTTLING);
2337 }
2338 
2339 /*
2340  * ixgbe_get_prop - Get a property value out of the configuration file
2341  * ixgbe.conf.
2342  *
2343  * Caller provides the name of the property, a default value, a minimum
2344  * value, and a maximum value.
2345  *
2346  * Return configured value of the property, with default, minimum and
2347  * maximum properly applied.
2348  */
2349 static int
2350 ixgbe_get_prop(ixgbe_t *ixgbe,
2351     char *propname,	/* name of the property */
2352     int minval,		/* minimum acceptable value */
2353     int maxval,		/* maximim acceptable value */
2354     int defval)		/* default value */
2355 {
2356 	int value;
2357 
2358 	/*
2359 	 * Call ddi_prop_get_int() to read the conf settings
2360 	 */
2361 	value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip,
2362 	    DDI_PROP_DONTPASS, propname, defval);
2363 	if (value > maxval)
2364 		value = maxval;
2365 
2366 	if (value < minval)
2367 		value = minval;
2368 
2369 	return (value);
2370 }
2371 
2372 /*
2373  * ixgbe_driver_setup_link - Using the link properties to setup the link.
2374  */
2375 int
2376 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw)
2377 {
2378 	struct ixgbe_mac_info *mac;
2379 	struct ixgbe_phy_info *phy;
2380 	boolean_t invalid;
2381 
2382 	mac = &ixgbe->hw.mac;
2383 	phy = &ixgbe->hw.phy;
2384 	invalid = B_FALSE;
2385 
2386 	if (ixgbe->param_adv_autoneg_cap == 1) {
2387 		mac->autoneg = B_TRUE;
2388 		phy->autoneg_advertised = 0;
2389 
2390 		/*
2391 		 * No half duplex support with 10Gb parts
2392 		 */
2393 		if (ixgbe->param_adv_10000fdx_cap == 1)
2394 			phy->autoneg_advertised |= IXGBE_LINK_SPEED_10GB_FULL;
2395 
2396 		if (ixgbe->param_adv_1000fdx_cap == 1)
2397 			phy->autoneg_advertised |= IXGBE_LINK_SPEED_1GB_FULL;
2398 
2399 		if (ixgbe->param_adv_100fdx_cap == 1)
2400 			phy->autoneg_advertised |= IXGBE_LINK_SPEED_100_FULL;
2401 
2402 		if (phy->autoneg_advertised == 0)
2403 			invalid = B_TRUE;
2404 	} else {
2405 		ixgbe->hw.mac.autoneg = B_FALSE;
2406 	}
2407 
2408 	if (invalid) {
2409 		ixgbe_notice(ixgbe, "Invalid link settings. Setup link to "
2410 		    "autonegotiation with full link capabilities.");
2411 		ixgbe->hw.mac.autoneg = B_TRUE;
2412 	}
2413 
2414 	if (setup_hw) {
2415 		if (ixgbe_setup_link(&ixgbe->hw) != IXGBE_SUCCESS)
2416 			return (IXGBE_FAILURE);
2417 	}
2418 
2419 	return (IXGBE_SUCCESS);
2420 }
2421 
2422 /*
2423  * ixgbe_driver_link_check - Link status processing.
2424  */
2425 static boolean_t
2426 ixgbe_driver_link_check(ixgbe_t *ixgbe)
2427 {
2428 	struct ixgbe_hw *hw = &ixgbe->hw;
2429 	ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
2430 	boolean_t link_up = B_FALSE;
2431 	boolean_t link_changed = B_FALSE;
2432 
2433 	ASSERT(mutex_owned(&ixgbe->gen_lock));
2434 
2435 	(void) ixgbe_check_link(hw, &speed, &link_up);
2436 	if (link_up) {
2437 		/*
2438 		 * The Link is up, check whether it was marked as down earlier
2439 		 */
2440 		if (ixgbe->link_state != LINK_STATE_UP) {
2441 			switch (speed) {
2442 				case IXGBE_LINK_SPEED_10GB_FULL:
2443 					ixgbe->link_speed = SPEED_10GB;
2444 					break;
2445 				case IXGBE_LINK_SPEED_1GB_FULL:
2446 					ixgbe->link_speed = SPEED_1GB;
2447 					break;
2448 				case IXGBE_LINK_SPEED_100_FULL:
2449 					ixgbe->link_speed = SPEED_100;
2450 			}
2451 			ixgbe->link_duplex = LINK_DUPLEX_FULL;
2452 			ixgbe->link_state = LINK_STATE_UP;
2453 			ixgbe->link_down_timeout = 0;
2454 			link_changed = B_TRUE;
2455 		}
2456 	} else {
2457 		if (ixgbe->link_state != LINK_STATE_DOWN) {
2458 			ixgbe->link_speed = 0;
2459 			ixgbe->link_duplex = 0;
2460 			ixgbe->link_state = LINK_STATE_DOWN;
2461 			link_changed = B_TRUE;
2462 		}
2463 
2464 		if (ixgbe->ixgbe_state & IXGBE_STARTED) {
2465 			if (ixgbe->link_down_timeout < MAX_LINK_DOWN_TIMEOUT) {
2466 				ixgbe->link_down_timeout++;
2467 			} else if (ixgbe->link_down_timeout ==
2468 			    MAX_LINK_DOWN_TIMEOUT) {
2469 				ixgbe_tx_clean(ixgbe);
2470 				ixgbe->link_down_timeout++;
2471 			}
2472 		}
2473 	}
2474 
2475 	return (link_changed);
2476 }
2477 
2478 /*
2479  * ixgbe_local_timer - Driver watchdog function.
2480  *
2481  * This function will handle the transmit stall check, link status check and
2482  * other routines.
2483  */
2484 static void
2485 ixgbe_local_timer(void *arg)
2486 {
2487 	ixgbe_t *ixgbe = (ixgbe_t *)arg;
2488 
2489 	if (ixgbe_stall_check(ixgbe)) {
2490 		ixgbe->reset_count++;
2491 		if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
2492 			ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
2493 	}
2494 
2495 	ixgbe_restart_watchdog_timer(ixgbe);
2496 }
2497 
2498 /*
2499  * ixgbe_stall_check - Check for transmit stall.
2500  *
2501  * This function checks if the adapter is stalled (in transmit).
2502  *
2503  * It is called each time the watchdog timeout is invoked.
2504  * If the transmit descriptor reclaim continuously fails,
2505  * the watchdog value will increment by 1. If the watchdog
2506  * value exceeds the threshold, the ixgbe is assumed to
2507  * have stalled and need to be reset.
2508  */
2509 static boolean_t
2510 ixgbe_stall_check(ixgbe_t *ixgbe)
2511 {
2512 	ixgbe_tx_ring_t *tx_ring;
2513 	boolean_t result;
2514 	int i;
2515 
2516 	if (ixgbe->link_state != LINK_STATE_UP)
2517 		return (B_FALSE);
2518 
2519 	/*
2520 	 * If any tx ring is stalled, we'll reset the chipset
2521 	 */
2522 	result = B_FALSE;
2523 	for (i = 0; i < ixgbe->num_tx_rings; i++) {
2524 		tx_ring = &ixgbe->tx_rings[i];
2525 		tx_ring->tx_recycle(tx_ring);
2526 
2527 		if (tx_ring->recycle_fail > 0)
2528 			tx_ring->stall_watchdog++;
2529 		else
2530 			tx_ring->stall_watchdog = 0;
2531 
2532 		if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) {
2533 			result = B_TRUE;
2534 			break;
2535 		}
2536 	}
2537 
2538 	if (result) {
2539 		tx_ring->stall_watchdog = 0;
2540 		tx_ring->recycle_fail = 0;
2541 	}
2542 
2543 	return (result);
2544 }
2545 
2546 
2547 /*
2548  * is_valid_mac_addr - Check if the mac address is valid.
2549  */
2550 static boolean_t
2551 is_valid_mac_addr(uint8_t *mac_addr)
2552 {
2553 	const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 };
2554 	const uint8_t addr_test2[6] =
2555 	    { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
2556 
2557 	if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) ||
2558 	    !(bcmp(addr_test2, mac_addr, ETHERADDRL)))
2559 		return (B_FALSE);
2560 
2561 	return (B_TRUE);
2562 }
2563 
2564 static boolean_t
2565 ixgbe_find_mac_address(ixgbe_t *ixgbe)
2566 {
2567 #ifdef __sparc
2568 	struct ixgbe_hw *hw = &ixgbe->hw;
2569 	uchar_t *bytes;
2570 	struct ether_addr sysaddr;
2571 	uint_t nelts;
2572 	int err;
2573 	boolean_t found = B_FALSE;
2574 
2575 	/*
2576 	 * The "vendor's factory-set address" may already have
2577 	 * been extracted from the chip, but if the property
2578 	 * "local-mac-address" is set we use that instead.
2579 	 *
2580 	 * We check whether it looks like an array of 6
2581 	 * bytes (which it should, if OBP set it).  If we can't
2582 	 * make sense of it this way, we'll ignore it.
2583 	 */
2584 	err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
2585 	    DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts);
2586 	if (err == DDI_PROP_SUCCESS) {
2587 		if (nelts == ETHERADDRL) {
2588 			while (nelts--)
2589 				hw->mac.addr[nelts] = bytes[nelts];
2590 			found = B_TRUE;
2591 		}
2592 		ddi_prop_free(bytes);
2593 	}
2594 
2595 	/*
2596 	 * Look up the OBP property "local-mac-address?". If the user has set
2597 	 * 'local-mac-address? = false', use "the system address" instead.
2598 	 */
2599 	if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0,
2600 	    "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
2601 		if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
2602 			if (localetheraddr(NULL, &sysaddr) != 0) {
2603 				bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
2604 				found = B_TRUE;
2605 			}
2606 		}
2607 		ddi_prop_free(bytes);
2608 	}
2609 
2610 	/*
2611 	 * Finally(!), if there's a valid "mac-address" property (created
2612 	 * if we netbooted from this interface), we must use this instead
2613 	 * of any of the above to ensure that the NFS/install server doesn't
2614 	 * get confused by the address changing as Solaris takes over!
2615 	 */
2616 	err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
2617 	    DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
2618 	if (err == DDI_PROP_SUCCESS) {
2619 		if (nelts == ETHERADDRL) {
2620 			while (nelts--)
2621 				hw->mac.addr[nelts] = bytes[nelts];
2622 			found = B_TRUE;
2623 		}
2624 		ddi_prop_free(bytes);
2625 	}
2626 
2627 	if (found) {
2628 		bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
2629 		return (B_TRUE);
2630 	}
2631 #else
2632 	_NOTE(ARGUNUSED(ixgbe));
2633 #endif
2634 
2635 	return (B_TRUE);
2636 }
2637 
2638 #pragma inline(ixgbe_arm_watchdog_timer)
2639 static void
2640 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe)
2641 {
2642 	/*
2643 	 * Fire a watchdog timer
2644 	 */
2645 	ixgbe->watchdog_tid =
2646 	    timeout(ixgbe_local_timer,
2647 	    (void *)ixgbe, 1 * drv_usectohz(1000000));
2648 
2649 }
2650 
2651 /*
2652  * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer.
2653  */
2654 void
2655 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe)
2656 {
2657 	mutex_enter(&ixgbe->watchdog_lock);
2658 
2659 	if (!ixgbe->watchdog_enable) {
2660 		ixgbe->watchdog_enable = B_TRUE;
2661 		ixgbe->watchdog_start = B_TRUE;
2662 		ixgbe_arm_watchdog_timer(ixgbe);
2663 	}
2664 
2665 	mutex_exit(&ixgbe->watchdog_lock);
2666 }
2667 
2668 /*
2669  * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer.
2670  */
2671 void
2672 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe)
2673 {
2674 	timeout_id_t tid;
2675 
2676 	mutex_enter(&ixgbe->watchdog_lock);
2677 
2678 	ixgbe->watchdog_enable = B_FALSE;
2679 	ixgbe->watchdog_start = B_FALSE;
2680 	tid = ixgbe->watchdog_tid;
2681 	ixgbe->watchdog_tid = 0;
2682 
2683 	mutex_exit(&ixgbe->watchdog_lock);
2684 
2685 	if (tid != 0)
2686 		(void) untimeout(tid);
2687 }
2688 
2689 /*
2690  * ixgbe_start_watchdog_timer - Start the driver watchdog timer.
2691  */
2692 static void
2693 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe)
2694 {
2695 	mutex_enter(&ixgbe->watchdog_lock);
2696 
2697 	if (ixgbe->watchdog_enable) {
2698 		if (!ixgbe->watchdog_start) {
2699 			ixgbe->watchdog_start = B_TRUE;
2700 			ixgbe_arm_watchdog_timer(ixgbe);
2701 		}
2702 	}
2703 
2704 	mutex_exit(&ixgbe->watchdog_lock);
2705 }
2706 
2707 /*
2708  * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer.
2709  */
2710 static void
2711 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe)
2712 {
2713 	mutex_enter(&ixgbe->watchdog_lock);
2714 
2715 	if (ixgbe->watchdog_start)
2716 		ixgbe_arm_watchdog_timer(ixgbe);
2717 
2718 	mutex_exit(&ixgbe->watchdog_lock);
2719 }
2720 
2721 /*
2722  * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer.
2723  */
2724 static void
2725 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe)
2726 {
2727 	timeout_id_t tid;
2728 
2729 	mutex_enter(&ixgbe->watchdog_lock);
2730 
2731 	ixgbe->watchdog_start = B_FALSE;
2732 	tid = ixgbe->watchdog_tid;
2733 	ixgbe->watchdog_tid = 0;
2734 
2735 	mutex_exit(&ixgbe->watchdog_lock);
2736 
2737 	if (tid != 0)
2738 		(void) untimeout(tid);
2739 }
2740 
2741 /*
2742  * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts.
2743  */
2744 static void
2745 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe)
2746 {
2747 	struct ixgbe_hw *hw = &ixgbe->hw;
2748 
2749 	/*
2750 	 * mask all interrupts off
2751 	 */
2752 	IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff);
2753 
2754 	/*
2755 	 * for MSI-X, also disable autoclear
2756 	 */
2757 	if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
2758 		IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0);
2759 	}
2760 
2761 	IXGBE_WRITE_FLUSH(hw);
2762 }
2763 
2764 /*
2765  * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts.
2766  */
2767 static void
2768 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe)
2769 {
2770 	struct ixgbe_hw *hw = &ixgbe->hw;
2771 	uint32_t eims, eiac, gpie;
2772 
2773 	gpie = 0;
2774 	eims = IXGBE_EIMS_ENABLE_MASK;	/* shared code default */
2775 	eims &= ~IXGBE_EIMS_TCP_TIMER;	/* minus tcp timer */
2776 
2777 	/*
2778 	 * msi-x mode
2779 	 */
2780 	if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
2781 		/* enable autoclear but not on bits 29:20 */
2782 		eiac = (eims & ~0x3ff00000);
2783 
2784 		/* general purpose interrupt enable */
2785 		gpie |= (IXGBE_GPIE_MSIX_MODE |
2786 		    IXGBE_GPIE_PBA_SUPPORT |IXGBE_GPIE_OCD);
2787 	/*
2788 	 * non-msi-x mode
2789 	 */
2790 	} else {
2791 
2792 		/* disable autoclear, leave gpie at default */
2793 		eiac = 0;
2794 	}
2795 
2796 	IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims);
2797 	IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac);
2798 	IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
2799 	IXGBE_WRITE_FLUSH(hw);
2800 }
2801 
2802 /*
2803  * ixgbe_loopback_ioctl - Loopback support.
2804  */
2805 enum ioc_reply
2806 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp)
2807 {
2808 	lb_info_sz_t *lbsp;
2809 	lb_property_t *lbpp;
2810 	uint32_t *lbmp;
2811 	uint32_t size;
2812 	uint32_t value;
2813 
2814 	if (mp->b_cont == NULL)
2815 		return (IOC_INVAL);
2816 
2817 	switch (iocp->ioc_cmd) {
2818 	default:
2819 		return (IOC_INVAL);
2820 
2821 	case LB_GET_INFO_SIZE:
2822 		size = sizeof (lb_info_sz_t);
2823 		if (iocp->ioc_count != size)
2824 			return (IOC_INVAL);
2825 
2826 		value = sizeof (lb_normal);
2827 		value += sizeof (lb_mac);
2828 
2829 		lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr;
2830 		*lbsp = value;
2831 		break;
2832 
2833 	case LB_GET_INFO:
2834 		value = sizeof (lb_normal);
2835 		value += sizeof (lb_mac);
2836 
2837 		size = value;
2838 		if (iocp->ioc_count != size)
2839 			return (IOC_INVAL);
2840 
2841 		value = 0;
2842 		lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr;
2843 
2844 		lbpp[value++] = lb_normal;
2845 		lbpp[value++] = lb_mac;
2846 		break;
2847 
2848 	case LB_GET_MODE:
2849 		size = sizeof (uint32_t);
2850 		if (iocp->ioc_count != size)
2851 			return (IOC_INVAL);
2852 
2853 		lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
2854 		*lbmp = ixgbe->loopback_mode;
2855 		break;
2856 
2857 	case LB_SET_MODE:
2858 		size = 0;
2859 		if (iocp->ioc_count != sizeof (uint32_t))
2860 			return (IOC_INVAL);
2861 
2862 		lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
2863 		if (!ixgbe_set_loopback_mode(ixgbe, *lbmp))
2864 			return (IOC_INVAL);
2865 		break;
2866 	}
2867 
2868 	iocp->ioc_count = size;
2869 	iocp->ioc_error = 0;
2870 
2871 	if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
2872 		ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
2873 		return (IOC_INVAL);
2874 	}
2875 
2876 	return (IOC_REPLY);
2877 }
2878 
2879 /*
2880  * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode.
2881  */
2882 static boolean_t
2883 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode)
2884 {
2885 	struct ixgbe_hw *hw;
2886 
2887 	if (mode == ixgbe->loopback_mode)
2888 		return (B_TRUE);
2889 
2890 	hw = &ixgbe->hw;
2891 
2892 	ixgbe->loopback_mode = mode;
2893 
2894 	if (mode == IXGBE_LB_NONE) {
2895 		/*
2896 		 * Reset the chip
2897 		 */
2898 		hw->phy.autoneg_wait_to_complete = B_TRUE;
2899 		(void) ixgbe_reset(ixgbe);
2900 		hw->phy.autoneg_wait_to_complete = B_FALSE;
2901 		return (B_TRUE);
2902 	}
2903 
2904 	mutex_enter(&ixgbe->gen_lock);
2905 
2906 	switch (mode) {
2907 	default:
2908 		mutex_exit(&ixgbe->gen_lock);
2909 		return (B_FALSE);
2910 
2911 	case IXGBE_LB_INTERNAL_MAC:
2912 		ixgbe_set_internal_mac_loopback(ixgbe);
2913 		break;
2914 	}
2915 
2916 	mutex_exit(&ixgbe->gen_lock);
2917 
2918 	return (B_TRUE);
2919 }
2920 
2921 /*
2922  * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode.
2923  */
2924 static void
2925 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe)
2926 {
2927 	struct ixgbe_hw *hw;
2928 	uint32_t reg;
2929 	uint8_t atlas;
2930 
2931 	hw = &ixgbe->hw;
2932 
2933 	/*
2934 	 * Setup MAC loopback
2935 	 */
2936 	reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0);
2937 	reg |= IXGBE_HLREG0_LPBK;
2938 	IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg);
2939 
2940 	reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
2941 	reg &= ~IXGBE_AUTOC_LMS_MASK;
2942 	IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
2943 
2944 	/*
2945 	 * Disable Atlas Tx lanes to keep packets in loopback and not on wire
2946 	 */
2947 	if (hw->mac.type == ixgbe_mac_82598EB) {
2948 		(void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
2949 		    &atlas);
2950 		atlas |= IXGBE_ATLAS_PDN_TX_REG_EN;
2951 		(void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
2952 		    atlas);
2953 
2954 		(void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
2955 		    &atlas);
2956 		atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL;
2957 		(void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
2958 		    atlas);
2959 
2960 		(void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
2961 		    &atlas);
2962 		atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL;
2963 		(void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
2964 		    atlas);
2965 
2966 		(void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
2967 		    &atlas);
2968 		atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL;
2969 		(void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
2970 		    atlas);
2971 	}
2972 }
2973 
2974 #pragma inline(ixgbe_intr_rx_work)
2975 /*
2976  * ixgbe_intr_rx_work - RX processing of ISR.
2977  */
2978 static void
2979 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring)
2980 {
2981 	mblk_t *mp;
2982 
2983 	mutex_enter(&rx_ring->rx_lock);
2984 
2985 	mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
2986 	mutex_exit(&rx_ring->rx_lock);
2987 
2988 	if (mp != NULL)
2989 		mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
2990 		    rx_ring->ring_gen_num);
2991 }
2992 
2993 #pragma inline(ixgbe_intr_tx_work)
2994 /*
2995  * ixgbe_intr_tx_work - TX processing of ISR.
2996  */
2997 static void
2998 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring)
2999 {
3000 	/*
3001 	 * Recycle the tx descriptors
3002 	 */
3003 	tx_ring->tx_recycle(tx_ring);
3004 
3005 	/*
3006 	 * Schedule the re-transmit
3007 	 */
3008 	if (tx_ring->reschedule &&
3009 	    (tx_ring->tbd_free >= tx_ring->resched_thresh)) {
3010 		tx_ring->reschedule = B_FALSE;
3011 		mac_tx_ring_update(tx_ring->ixgbe->mac_hdl,
3012 		    tx_ring->ring_handle);
3013 		IXGBE_DEBUG_STAT(tx_ring->stat_reschedule);
3014 	}
3015 }
3016 
3017 #pragma inline(ixgbe_intr_other_work)
3018 /*
3019  * ixgbe_intr_other_work - Other processing of ISR.
3020  */
3021 static void
3022 ixgbe_intr_other_work(ixgbe_t *ixgbe)
3023 {
3024 	boolean_t link_changed;
3025 
3026 	ixgbe_stop_watchdog_timer(ixgbe);
3027 
3028 	mutex_enter(&ixgbe->gen_lock);
3029 
3030 	/*
3031 	 * Take care of link status change
3032 	 */
3033 	link_changed = ixgbe_driver_link_check(ixgbe);
3034 
3035 	/*
3036 	 * Get new phy state
3037 	 */
3038 	ixgbe_get_hw_state(ixgbe);
3039 
3040 	mutex_exit(&ixgbe->gen_lock);
3041 
3042 	if (link_changed)
3043 		mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
3044 
3045 	ixgbe_start_watchdog_timer(ixgbe);
3046 }
3047 
3048 /*
3049  * ixgbe_intr_legacy - Interrupt handler for legacy interrupts.
3050  */
3051 static uint_t
3052 ixgbe_intr_legacy(void *arg1, void *arg2)
3053 {
3054 	_NOTE(ARGUNUSED(arg2));
3055 	ixgbe_t *ixgbe = (ixgbe_t *)arg1;
3056 	struct ixgbe_hw *hw = &ixgbe->hw;
3057 	ixgbe_tx_ring_t *tx_ring;
3058 	ixgbe_rx_ring_t *rx_ring;
3059 	uint32_t eicr;
3060 	mblk_t *mp;
3061 	boolean_t tx_reschedule;
3062 	boolean_t link_changed;
3063 	uint_t result;
3064 
3065 
3066 	mutex_enter(&ixgbe->gen_lock);
3067 
3068 	if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
3069 		mutex_exit(&ixgbe->gen_lock);
3070 		return (DDI_INTR_UNCLAIMED);
3071 	}
3072 
3073 	mp = NULL;
3074 	tx_reschedule = B_FALSE;
3075 	link_changed = B_FALSE;
3076 
3077 	/*
3078 	 * Any bit set in eicr: claim this interrupt
3079 	 */
3080 	eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
3081 	if (eicr) {
3082 		/*
3083 		 * For legacy interrupt, we have only one interrupt,
3084 		 * so we have only one rx ring and one tx ring enabled.
3085 		 */
3086 		ASSERT(ixgbe->num_rx_rings == 1);
3087 		ASSERT(ixgbe->num_tx_rings == 1);
3088 
3089 		/*
3090 		 * For legacy interrupt, rx rings[0] will use RTxQ[0].
3091 		 */
3092 		if (eicr & 0x1) {
3093 			/*
3094 			 * Clean the rx descriptors
3095 			 */
3096 			rx_ring = &ixgbe->rx_rings[0];
3097 			mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
3098 		}
3099 
3100 		/*
3101 		 * For legacy interrupt, tx rings[0] will use RTxQ[1].
3102 		 */
3103 		if (eicr & 0x2) {
3104 			/*
3105 			 * Recycle the tx descriptors
3106 			 */
3107 			tx_ring = &ixgbe->tx_rings[0];
3108 			tx_ring->tx_recycle(tx_ring);
3109 
3110 			/*
3111 			 * Schedule the re-transmit
3112 			 */
3113 			tx_reschedule = (tx_ring->reschedule &&
3114 			    (tx_ring->tbd_free >= tx_ring->resched_thresh));
3115 		}
3116 
3117 		if (eicr & IXGBE_EICR_LSC) {
3118 
3119 			/* take care of link status change */
3120 			link_changed = ixgbe_driver_link_check(ixgbe);
3121 
3122 			/* Get new phy state */
3123 			ixgbe_get_hw_state(ixgbe);
3124 		}
3125 
3126 		result = DDI_INTR_CLAIMED;
3127 	} else {
3128 		/*
3129 		 * No interrupt cause bits set: don't claim this interrupt.
3130 		 */
3131 		result = DDI_INTR_UNCLAIMED;
3132 	}
3133 
3134 	mutex_exit(&ixgbe->gen_lock);
3135 
3136 	/*
3137 	 * Do the following work outside of the gen_lock
3138 	 */
3139 	if (mp != NULL)
3140 		mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
3141 		    rx_ring->ring_gen_num);
3142 
3143 	if (tx_reschedule)  {
3144 		tx_ring->reschedule = B_FALSE;
3145 		mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle);
3146 		IXGBE_DEBUG_STAT(tx_ring->stat_reschedule);
3147 	}
3148 
3149 	if (link_changed)
3150 		mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
3151 
3152 	return (result);
3153 }
3154 
3155 /*
3156  * ixgbe_intr_msi - Interrupt handler for MSI.
3157  */
3158 static uint_t
3159 ixgbe_intr_msi(void *arg1, void *arg2)
3160 {
3161 	_NOTE(ARGUNUSED(arg2));
3162 	ixgbe_t *ixgbe = (ixgbe_t *)arg1;
3163 	struct ixgbe_hw *hw = &ixgbe->hw;
3164 	uint32_t eicr;
3165 
3166 	eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
3167 
3168 	/*
3169 	 * For MSI interrupt, we have only one vector,
3170 	 * so we have only one rx ring and one tx ring enabled.
3171 	 */
3172 	ASSERT(ixgbe->num_rx_rings == 1);
3173 	ASSERT(ixgbe->num_tx_rings == 1);
3174 
3175 	/*
3176 	 * For MSI interrupt, rx rings[0] will use RTxQ[0].
3177 	 */
3178 	if (eicr & 0x1) {
3179 		ixgbe_intr_rx_work(&ixgbe->rx_rings[0]);
3180 	}
3181 
3182 	/*
3183 	 * For MSI interrupt, tx rings[0] will use RTxQ[1].
3184 	 */
3185 	if (eicr & 0x2) {
3186 		ixgbe_intr_tx_work(&ixgbe->tx_rings[0]);
3187 	}
3188 
3189 	if (eicr & IXGBE_EICR_LSC) {
3190 		ixgbe_intr_other_work(ixgbe);
3191 	}
3192 
3193 	return (DDI_INTR_CLAIMED);
3194 }
3195 
3196 /*
3197  * ixgbe_intr_rx_tx - Interrupt handler for rx and tx.
3198  */
3199 static uint_t
3200 ixgbe_intr_rx_tx(void *arg1, void *arg2)
3201 {
3202 	_NOTE(ARGUNUSED(arg2));
3203 	ixgbe_ring_vector_t *vect = (ixgbe_ring_vector_t *)arg1;
3204 	ixgbe_t *ixgbe = vect->ixgbe;
3205 	int r_idx = 0;
3206 
3207 	/*
3208 	 * Clean each rx ring that has its bit set in the map
3209 	 */
3210 	r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1));
3211 	while (r_idx >= 0) {
3212 		ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]);
3213 		r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
3214 		    (ixgbe->num_rx_rings - 1));
3215 	}
3216 
3217 	/*
3218 	 * Clean each tx ring that has its bit set in the map
3219 	 */
3220 	r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1));
3221 	while (r_idx >= 0) {
3222 		ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]);
3223 		r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
3224 		    (ixgbe->num_tx_rings - 1));
3225 	}
3226 
3227 	return (DDI_INTR_CLAIMED);
3228 }
3229 
3230 /*
3231  * ixgbe_intr_other - Interrupt handler for other.
3232  *
3233  * Only look for other work if the right bits are set in the
3234  * Interrupt Cause Register.
3235  */
3236 static uint_t
3237 ixgbe_intr_other(void *arg1, void *arg2)
3238 {
3239 	_NOTE(ARGUNUSED(arg2));
3240 	ixgbe_t *ixgbe = (ixgbe_t *)arg1;
3241 	struct ixgbe_hw *hw = &ixgbe->hw;
3242 	uint32_t eicr;
3243 
3244 	eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
3245 
3246 	/*
3247 	 * Need check cause bits and only link change will
3248 	 * be processed
3249 	 */
3250 	if (eicr & IXGBE_EICR_LSC) {
3251 		ixgbe_intr_other_work(ixgbe);
3252 	}
3253 
3254 	return (DDI_INTR_CLAIMED);
3255 }
3256 
3257 /*
3258  * ixgbe_alloc_intrs - Allocate interrupts for the driver.
3259  *
3260  * Normal sequence is to try MSI-X; if not sucessful, try MSI;
3261  * if not successful, try Legacy.
3262  * ixgbe->intr_force can be used to force sequence to start with
3263  * any of the 3 types.
3264  * If MSI-X is not used, number of tx/rx rings is forced to 1.
3265  */
3266 static int
3267 ixgbe_alloc_intrs(ixgbe_t *ixgbe)
3268 {
3269 	dev_info_t *devinfo;
3270 	int intr_types;
3271 	int rc;
3272 
3273 	devinfo = ixgbe->dip;
3274 
3275 	/*
3276 	 * Get supported interrupt types
3277 	 */
3278 	rc = ddi_intr_get_supported_types(devinfo, &intr_types);
3279 
3280 	if (rc != DDI_SUCCESS) {
3281 		ixgbe_log(ixgbe,
3282 		    "Get supported interrupt types failed: %d", rc);
3283 		return (IXGBE_FAILURE);
3284 	}
3285 	IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types);
3286 
3287 	ixgbe->intr_type = 0;
3288 
3289 	/*
3290 	 * Install MSI-X interrupts
3291 	 */
3292 	if ((intr_types & DDI_INTR_TYPE_MSIX) &&
3293 	    (ixgbe->intr_force <= IXGBE_INTR_MSIX)) {
3294 		rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX);
3295 		if (rc == IXGBE_SUCCESS)
3296 			return (IXGBE_SUCCESS);
3297 
3298 		ixgbe_log(ixgbe,
3299 		    "Allocate MSI-X failed, trying MSI interrupts...");
3300 	}
3301 
3302 	/*
3303 	 * MSI-X not used, force rings and groups to 1
3304 	 */
3305 	ixgbe->num_rx_rings = 1;
3306 	ixgbe->num_rx_groups = 1;
3307 	ixgbe->num_tx_rings = 1;
3308 	ixgbe_log(ixgbe,
3309 	    "MSI-X not used, force rings and groups number to 1");
3310 
3311 	/*
3312 	 * Install MSI interrupts
3313 	 */
3314 	if ((intr_types & DDI_INTR_TYPE_MSI) &&
3315 	    (ixgbe->intr_force <= IXGBE_INTR_MSI)) {
3316 		rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI);
3317 		if (rc == IXGBE_SUCCESS)
3318 			return (IXGBE_SUCCESS);
3319 
3320 		ixgbe_log(ixgbe,
3321 		    "Allocate MSI failed, trying Legacy interrupts...");
3322 	}
3323 
3324 	/*
3325 	 * Install legacy interrupts
3326 	 */
3327 	if (intr_types & DDI_INTR_TYPE_FIXED) {
3328 		rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED);
3329 		if (rc == IXGBE_SUCCESS)
3330 			return (IXGBE_SUCCESS);
3331 
3332 		ixgbe_log(ixgbe,
3333 		    "Allocate Legacy interrupts failed");
3334 	}
3335 
3336 	/*
3337 	 * If none of the 3 types succeeded, return failure
3338 	 */
3339 	return (IXGBE_FAILURE);
3340 }
3341 
3342 /*
3343  * ixgbe_alloc_intr_handles - Allocate interrupt handles.
3344  *
3345  * For legacy and MSI, only 1 handle is needed.  For MSI-X,
3346  * if fewer than 2 handles are available, return failure.
3347  * Upon success, this maps the vectors to rx and tx rings for
3348  * interrupts.
3349  */
3350 static int
3351 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type)
3352 {
3353 	dev_info_t *devinfo;
3354 	int request, count, avail, actual;
3355 	int minimum;
3356 	int rc;
3357 
3358 	devinfo = ixgbe->dip;
3359 
3360 	switch (intr_type) {
3361 	case DDI_INTR_TYPE_FIXED:
3362 		request = 1;	/* Request 1 legacy interrupt handle */
3363 		minimum = 1;
3364 		IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy");
3365 		break;
3366 
3367 	case DDI_INTR_TYPE_MSI:
3368 		request = 1;	/* Request 1 MSI interrupt handle */
3369 		minimum = 1;
3370 		IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI");
3371 		break;
3372 
3373 	case DDI_INTR_TYPE_MSIX:
3374 		/*
3375 		 * Best number of vectors for the adapter is
3376 		 * # rx rings + # tx rings + 1 for other.
3377 		 */
3378 		request = ixgbe->num_rx_rings + ixgbe->num_tx_rings + 1;
3379 		if (request > (IXGBE_MAX_RING_VECTOR + 1))
3380 			request = IXGBE_MAX_RING_VECTOR + 1;
3381 		minimum = 2;
3382 		IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X");
3383 		break;
3384 
3385 	default:
3386 		ixgbe_log(ixgbe,
3387 		    "invalid call to ixgbe_alloc_intr_handles(): %d\n",
3388 		    intr_type);
3389 		return (IXGBE_FAILURE);
3390 	}
3391 	IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d  minimum: %d",
3392 	    request, minimum);
3393 
3394 	/*
3395 	 * Get number of supported interrupts
3396 	 */
3397 	rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
3398 	if ((rc != DDI_SUCCESS) || (count < minimum)) {
3399 		ixgbe_log(ixgbe,
3400 		    "Get interrupt number failed. Return: %d, count: %d",
3401 		    rc, count);
3402 		return (IXGBE_FAILURE);
3403 	}
3404 	IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count);
3405 
3406 	/*
3407 	 * Get number of available interrupts
3408 	 */
3409 	rc = ddi_intr_get_navail(devinfo, intr_type, &avail);
3410 	if ((rc != DDI_SUCCESS) || (avail < minimum)) {
3411 		ixgbe_log(ixgbe,
3412 		    "Get interrupt available number failed. "
3413 		    "Return: %d, available: %d", rc, avail);
3414 		return (IXGBE_FAILURE);
3415 	}
3416 	IXGBE_DEBUGLOG_1(ixgbe, "interrupts available: %d", avail);
3417 
3418 	if (avail < request) {
3419 		ixgbe_log(ixgbe, "Request %d handles, %d available",
3420 		    request, avail);
3421 		request = avail;
3422 	}
3423 
3424 	actual = 0;
3425 	ixgbe->intr_cnt = 0;
3426 
3427 	/*
3428 	 * Allocate an array of interrupt handles
3429 	 */
3430 	ixgbe->intr_size = request * sizeof (ddi_intr_handle_t);
3431 	ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP);
3432 
3433 	rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0,
3434 	    request, &actual, DDI_INTR_ALLOC_NORMAL);
3435 	if (rc != DDI_SUCCESS) {
3436 		ixgbe_log(ixgbe, "Allocate interrupts failed. "
3437 		    "return: %d, request: %d, actual: %d",
3438 		    rc, request, actual);
3439 		goto alloc_handle_fail;
3440 	}
3441 	IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual);
3442 
3443 	ixgbe->intr_cnt = actual;
3444 
3445 	/*
3446 	 * Now we know the actual number of vectors.  Here we map the vector
3447 	 * to other, rx rings and tx ring.
3448 	 */
3449 	if (actual < minimum) {
3450 		ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d",
3451 		    actual);
3452 		goto alloc_handle_fail;
3453 	}
3454 
3455 	/*
3456 	 * Get priority for first vector, assume remaining are all the same
3457 	 */
3458 	rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
3459 	if (rc != DDI_SUCCESS) {
3460 		ixgbe_log(ixgbe,
3461 		    "Get interrupt priority failed: %d", rc);
3462 		goto alloc_handle_fail;
3463 	}
3464 
3465 	rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
3466 	if (rc != DDI_SUCCESS) {
3467 		ixgbe_log(ixgbe,
3468 		    "Get interrupt cap failed: %d", rc);
3469 		goto alloc_handle_fail;
3470 	}
3471 
3472 	ixgbe->intr_type = intr_type;
3473 
3474 	return (IXGBE_SUCCESS);
3475 
3476 alloc_handle_fail:
3477 	ixgbe_rem_intrs(ixgbe);
3478 
3479 	return (IXGBE_FAILURE);
3480 }
3481 
3482 /*
3483  * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type.
3484  *
3485  * Before adding the interrupt handlers, the interrupt vectors have
3486  * been allocated, and the rx/tx rings have also been allocated.
3487  */
3488 static int
3489 ixgbe_add_intr_handlers(ixgbe_t *ixgbe)
3490 {
3491 	int vector = 0;
3492 	int rc;
3493 
3494 	switch (ixgbe->intr_type) {
3495 	case DDI_INTR_TYPE_MSIX:
3496 		/*
3497 		 * Add interrupt handler for rx and tx rings: vector[0 -
3498 		 * (ixgbe->intr_cnt -1)].
3499 		 */
3500 		for (vector = 0; vector < (ixgbe->intr_cnt -1); vector++) {
3501 			/*
3502 			 * install pointer to vect_map[vector]
3503 			 */
3504 			rc = ddi_intr_add_handler(ixgbe->htable[vector],
3505 			    (ddi_intr_handler_t *)ixgbe_intr_rx_tx,
3506 			    (void *)&ixgbe->vect_map[vector], NULL);
3507 
3508 			if (rc != DDI_SUCCESS) {
3509 				ixgbe_log(ixgbe,
3510 				    "Add rx interrupt handler failed. "
3511 				    "return: %d, vector: %d", rc, vector);
3512 				for (vector--; vector >= 0; vector--) {
3513 					(void) ddi_intr_remove_handler(
3514 					    ixgbe->htable[vector]);
3515 				}
3516 				return (IXGBE_FAILURE);
3517 			}
3518 		}
3519 
3520 		/*
3521 		 * Add interrupt handler for other: vector[ixgbe->intr_cnt -1]
3522 		 */
3523 		rc = ddi_intr_add_handler(ixgbe->htable[vector],
3524 		    (ddi_intr_handler_t *)ixgbe_intr_other,
3525 		    (void *)ixgbe, NULL);
3526 		if (rc != DDI_SUCCESS) {
3527 			ixgbe_log(ixgbe,
3528 			    "Add other interrupt handler failed: %d", rc);
3529 			return (IXGBE_FAILURE);
3530 		}
3531 
3532 		break;
3533 
3534 	case DDI_INTR_TYPE_MSI:
3535 		/*
3536 		 * Add interrupt handlers for the only vector
3537 		 */
3538 		rc = ddi_intr_add_handler(ixgbe->htable[vector],
3539 		    (ddi_intr_handler_t *)ixgbe_intr_msi,
3540 		    (void *)ixgbe, NULL);
3541 
3542 		if (rc != DDI_SUCCESS) {
3543 			ixgbe_log(ixgbe,
3544 			    "Add MSI interrupt handler failed: %d", rc);
3545 			return (IXGBE_FAILURE);
3546 		}
3547 
3548 		break;
3549 
3550 	case DDI_INTR_TYPE_FIXED:
3551 		/*
3552 		 * Add interrupt handlers for the only vector
3553 		 */
3554 		rc = ddi_intr_add_handler(ixgbe->htable[vector],
3555 		    (ddi_intr_handler_t *)ixgbe_intr_legacy,
3556 		    (void *)ixgbe, NULL);
3557 
3558 		if (rc != DDI_SUCCESS) {
3559 			ixgbe_log(ixgbe,
3560 			    "Add legacy interrupt handler failed: %d", rc);
3561 			return (IXGBE_FAILURE);
3562 		}
3563 
3564 		break;
3565 
3566 	default:
3567 		return (IXGBE_FAILURE);
3568 	}
3569 
3570 	ASSERT(vector == (ixgbe->intr_cnt -1));
3571 
3572 	return (IXGBE_SUCCESS);
3573 }
3574 
3575 #pragma inline(ixgbe_map_rxring_to_vector)
3576 /*
3577  * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector.
3578  */
3579 static void
3580 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx)
3581 {
3582 	ixgbe->vect_map[v_idx].ixgbe = ixgbe;
3583 
3584 	/*
3585 	 * Set bit in map
3586 	 */
3587 	BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
3588 
3589 	/*
3590 	 * Count bits set
3591 	 */
3592 	ixgbe->vect_map[v_idx].rxr_cnt++;
3593 
3594 	/*
3595 	 * Remember bit position
3596 	 */
3597 	ixgbe->rx_rings[r_idx].intr_vector = v_idx;
3598 	ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx;
3599 }
3600 
3601 #pragma inline(ixgbe_map_txring_to_vector)
3602 /*
3603  * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector.
3604  */
3605 static void
3606 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx)
3607 {
3608 	ixgbe->vect_map[v_idx].ixgbe = ixgbe;
3609 
3610 	/*
3611 	 * Set bit in map
3612 	 */
3613 	BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx);
3614 
3615 	/*
3616 	 * Count bits set
3617 	 */
3618 	ixgbe->vect_map[v_idx].txr_cnt++;
3619 
3620 	/*
3621 	 * Remember bit position
3622 	 */
3623 	ixgbe->tx_rings[t_idx].intr_vector = v_idx;
3624 	ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx;
3625 }
3626 
3627 /*
3628  * ixgbe_setup_ivar - Set the given entry in the given interrupt vector
3629  * allocation register (IVAR).
3630  */
3631 static void
3632 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector)
3633 {
3634 	struct ixgbe_hw *hw = &ixgbe->hw;
3635 	u32 ivar, index;
3636 
3637 	msix_vector |= IXGBE_IVAR_ALLOC_VAL;
3638 	index = (intr_alloc_entry >> 2) & 0x1F;
3639 	ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
3640 	ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3)));
3641 	ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3)));
3642 	IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
3643 }
3644 
3645 /*
3646  * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of
3647  * given interrupt vector allocation register (IVAR).
3648  */
3649 static void
3650 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry)
3651 {
3652 	struct ixgbe_hw *hw = &ixgbe->hw;
3653 	u32 ivar, index;
3654 
3655 	index = (intr_alloc_entry >> 2) & 0x1F;
3656 	ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
3657 	ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 * (intr_alloc_entry & 0x3)));
3658 	IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
3659 }
3660 
3661 /*
3662  * ixgbe_enable_ivar - Disble the given entry by clearing the VAL bit of
3663  * given interrupt vector allocation register (IVAR).
3664  */
3665 static void
3666 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry)
3667 {
3668 	struct ixgbe_hw *hw = &ixgbe->hw;
3669 	u32 ivar, index;
3670 
3671 	index = (intr_alloc_entry >> 2) & 0x1F;
3672 	ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
3673 	ivar &= ~(IXGBE_IVAR_ALLOC_VAL << (8 * (intr_alloc_entry & 0x3)));
3674 	IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
3675 }
3676 
3677 /*
3678  * ixgbe_map_rings_to_vectors - Map descriptor rings to interrupt vectors.
3679  *
3680  * For MSI-X, here will map rx and tx ring to vector[0 - (vectors -1)].
3681  * The last vector will be used for other interrupt.
3682  */
3683 static int
3684 ixgbe_map_rings_to_vectors(ixgbe_t *ixgbe)
3685 {
3686 	int i, vector = 0;
3687 
3688 	/* initialize vector map */
3689 	bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
3690 
3691 	/*
3692 	 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0],
3693 	 * tx rings[0] on RTxQ[1].
3694 	 */
3695 	if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
3696 		ixgbe_map_rxring_to_vector(ixgbe, 0, 0);
3697 		ixgbe_map_txring_to_vector(ixgbe, 0, 1);
3698 		return (IXGBE_SUCCESS);
3699 	}
3700 
3701 	/*
3702 	 * Ring/vector mapping for MSI-X
3703 	 */
3704 
3705 	/*
3706 	 * Map vectors to rx rings
3707 	 */
3708 	for (i = 0; i < ixgbe->num_rx_rings; i++) {
3709 		ixgbe_map_rxring_to_vector(ixgbe, i, vector);
3710 		vector = (vector +1) % (ixgbe->intr_cnt -1);
3711 	}
3712 
3713 	/*
3714 	 * Map vectors to tx rings
3715 	 */
3716 	for (i = 0; i < ixgbe->num_tx_rings; i++) {
3717 		ixgbe_map_txring_to_vector(ixgbe, i, vector);
3718 		vector = (vector +1) % (ixgbe->intr_cnt -1);
3719 	}
3720 
3721 	return (IXGBE_SUCCESS);
3722 }
3723 
3724 /*
3725  * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s).
3726  *
3727  * This relies on ring/vector mapping already set up in the
3728  * vect_map[] structures
3729  */
3730 static void
3731 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe)
3732 {
3733 	struct ixgbe_hw *hw = &ixgbe->hw;
3734 	ixgbe_ring_vector_t *vect;	/* vector bitmap */
3735 	int r_idx;	/* ring index */
3736 	int v_idx;	/* vector index */
3737 
3738 	/*
3739 	 * Clear any previous entries
3740 	 */
3741 	for (v_idx = 0; v_idx < IXGBE_IVAR_REG_NUM; v_idx++)
3742 		IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
3743 
3744 	/*
3745 	 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and
3746 	 * tx rings[0] will use RTxQ[1].
3747 	 */
3748 	if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
3749 		ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_RX_QUEUE(0), 0);
3750 		ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_TX_QUEUE(0), 1);
3751 		return;
3752 	}
3753 
3754 	/*
3755 	 * For MSI-X interrupt, "Other" is always on last vector.
3756 	 */
3757 	ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX,
3758 	    (ixgbe->intr_cnt - 1));
3759 
3760 	/*
3761 	 * For each interrupt vector, populate the IVAR table
3762 	 */
3763 	for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) {
3764 		vect = &ixgbe->vect_map[v_idx];
3765 
3766 		/*
3767 		 * For each rx ring bit set
3768 		 */
3769 		r_idx = bt_getlowbit(vect->rx_map, 0,
3770 		    (ixgbe->num_rx_rings - 1));
3771 
3772 		while (r_idx >= 0) {
3773 			ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_RX_QUEUE(r_idx),
3774 			    v_idx);
3775 			r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
3776 			    (ixgbe->num_rx_rings - 1));
3777 		}
3778 
3779 		/*
3780 		 * For each tx ring bit set
3781 		 */
3782 		r_idx = bt_getlowbit(vect->tx_map, 0,
3783 		    (ixgbe->num_tx_rings - 1));
3784 
3785 		while (r_idx >= 0) {
3786 			ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_TX_QUEUE(r_idx),
3787 			    v_idx);
3788 			r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
3789 			    (ixgbe->num_tx_rings - 1));
3790 		}
3791 	}
3792 }
3793 
3794 /*
3795  * ixgbe_rem_intr_handlers - Remove the interrupt handlers.
3796  */
3797 static void
3798 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe)
3799 {
3800 	int i;
3801 	int rc;
3802 
3803 	for (i = 0; i < ixgbe->intr_cnt; i++) {
3804 		rc = ddi_intr_remove_handler(ixgbe->htable[i]);
3805 		if (rc != DDI_SUCCESS) {
3806 			IXGBE_DEBUGLOG_1(ixgbe,
3807 			    "Remove intr handler failed: %d", rc);
3808 		}
3809 	}
3810 }
3811 
3812 /*
3813  * ixgbe_rem_intrs - Remove the allocated interrupts.
3814  */
3815 static void
3816 ixgbe_rem_intrs(ixgbe_t *ixgbe)
3817 {
3818 	int i;
3819 	int rc;
3820 
3821 	for (i = 0; i < ixgbe->intr_cnt; i++) {
3822 		rc = ddi_intr_free(ixgbe->htable[i]);
3823 		if (rc != DDI_SUCCESS) {
3824 			IXGBE_DEBUGLOG_1(ixgbe,
3825 			    "Free intr failed: %d", rc);
3826 		}
3827 	}
3828 
3829 	kmem_free(ixgbe->htable, ixgbe->intr_size);
3830 	ixgbe->htable = NULL;
3831 }
3832 
3833 /*
3834  * ixgbe_enable_intrs - Enable all the ddi interrupts.
3835  */
3836 static int
3837 ixgbe_enable_intrs(ixgbe_t *ixgbe)
3838 {
3839 	int i;
3840 	int rc;
3841 
3842 	/*
3843 	 * Enable interrupts
3844 	 */
3845 	if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
3846 		/*
3847 		 * Call ddi_intr_block_enable() for MSI
3848 		 */
3849 		rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt);
3850 		if (rc != DDI_SUCCESS) {
3851 			ixgbe_log(ixgbe,
3852 			    "Enable block intr failed: %d", rc);
3853 			return (IXGBE_FAILURE);
3854 		}
3855 	} else {
3856 		/*
3857 		 * Call ddi_intr_enable() for Legacy/MSI non block enable
3858 		 */
3859 		for (i = 0; i < ixgbe->intr_cnt; i++) {
3860 			rc = ddi_intr_enable(ixgbe->htable[i]);
3861 			if (rc != DDI_SUCCESS) {
3862 				ixgbe_log(ixgbe,
3863 				    "Enable intr failed: %d", rc);
3864 				return (IXGBE_FAILURE);
3865 			}
3866 		}
3867 	}
3868 
3869 	return (IXGBE_SUCCESS);
3870 }
3871 
3872 /*
3873  * ixgbe_disable_intrs - Disable all the interrupts.
3874  */
3875 static int
3876 ixgbe_disable_intrs(ixgbe_t *ixgbe)
3877 {
3878 	int i;
3879 	int rc;
3880 
3881 	/*
3882 	 * Disable all interrupts
3883 	 */
3884 	if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
3885 		rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt);
3886 		if (rc != DDI_SUCCESS) {
3887 			ixgbe_log(ixgbe,
3888 			    "Disable block intr failed: %d", rc);
3889 			return (IXGBE_FAILURE);
3890 		}
3891 	} else {
3892 		for (i = 0; i < ixgbe->intr_cnt; i++) {
3893 			rc = ddi_intr_disable(ixgbe->htable[i]);
3894 			if (rc != DDI_SUCCESS) {
3895 				ixgbe_log(ixgbe,
3896 				    "Disable intr failed: %d", rc);
3897 				return (IXGBE_FAILURE);
3898 			}
3899 		}
3900 	}
3901 
3902 	return (IXGBE_SUCCESS);
3903 }
3904 
3905 /*
3906  * ixgbe_get_hw_state - Get and save parameters related to adapter hardware.
3907  */
3908 static void
3909 ixgbe_get_hw_state(ixgbe_t *ixgbe)
3910 {
3911 	struct ixgbe_hw *hw = &ixgbe->hw;
3912 	uint32_t links;
3913 	uint32_t pcs1g_anlp = 0;
3914 	uint32_t pcs1g_ana = 0;
3915 
3916 	ASSERT(mutex_owned(&ixgbe->gen_lock));
3917 	ixgbe->param_lp_1000fdx_cap = 0;
3918 	ixgbe->param_lp_100fdx_cap  = 0;
3919 
3920 	links = IXGBE_READ_REG(hw, IXGBE_LINKS);
3921 	if (links & IXGBE_LINKS_PCS_1G_EN) {
3922 		pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP);
3923 		pcs1g_ana = IXGBE_READ_REG(hw, IXGBE_PCS1GANA);
3924 
3925 		ixgbe->param_lp_1000fdx_cap =
3926 		    (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
3927 		ixgbe->param_lp_100fdx_cap =
3928 		    (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
3929 	}
3930 
3931 	ixgbe->param_1000fdx_cap = (pcs1g_ana & IXGBE_PCS1GANA_FDC)  ? 1 : 0;
3932 	ixgbe->param_100fdx_cap = (pcs1g_ana & IXGBE_PCS1GANA_FDC)  ? 1 : 0;
3933 }
3934 
3935 /*
3936  * ixgbe_get_driver_control - Notify that driver is in control of device.
3937  */
3938 static void
3939 ixgbe_get_driver_control(struct ixgbe_hw *hw)
3940 {
3941 	uint32_t ctrl_ext;
3942 
3943 	/*
3944 	 * Notify firmware that driver is in control of device
3945 	 */
3946 	ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
3947 	ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
3948 	IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
3949 }
3950 
3951 /*
3952  * ixgbe_release_driver_control - Notify that driver is no longer in control
3953  * of device.
3954  */
3955 static void
3956 ixgbe_release_driver_control(struct ixgbe_hw *hw)
3957 {
3958 	uint32_t ctrl_ext;
3959 
3960 	/*
3961 	 * Notify firmware that driver is no longer in control of device
3962 	 */
3963 	ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
3964 	ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
3965 	IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
3966 }
3967 
3968 /*
3969  * ixgbe_atomic_reserve - Atomic decrease operation.
3970  */
3971 int
3972 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n)
3973 {
3974 	uint32_t oldval;
3975 	uint32_t newval;
3976 
3977 	/*
3978 	 * ATOMICALLY
3979 	 */
3980 	do {
3981 		oldval = *count_p;
3982 		if (oldval < n)
3983 			return (-1);
3984 		newval = oldval - n;
3985 	} while (atomic_cas_32(count_p, oldval, newval) != oldval);
3986 
3987 	return (newval);
3988 }
3989 
3990 /*
3991  * ixgbe_mc_table_itr - Traverse the entries in the multicast table.
3992  */
3993 static uint8_t *
3994 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq)
3995 {
3996 	_NOTE(ARGUNUSED(hw));
3997 	_NOTE(ARGUNUSED(vmdq));
3998 	uint8_t *addr = *upd_ptr;
3999 	uint8_t *new_ptr;
4000 
4001 	new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
4002 	*upd_ptr = new_ptr;
4003 	return (addr);
4004 }
4005 
4006 /*
4007  * FMA support
4008  */
4009 int
4010 ixgbe_check_acc_handle(ddi_acc_handle_t handle)
4011 {
4012 	ddi_fm_error_t de;
4013 
4014 	ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
4015 	ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
4016 	return (de.fme_status);
4017 }
4018 
4019 int
4020 ixgbe_check_dma_handle(ddi_dma_handle_t handle)
4021 {
4022 	ddi_fm_error_t de;
4023 
4024 	ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
4025 	return (de.fme_status);
4026 }
4027 
4028 /*
4029  * ixgbe_fm_error_cb - The IO fault service error handling callback function.
4030  */
4031 static int
4032 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
4033 {
4034 	_NOTE(ARGUNUSED(impl_data));
4035 	/*
4036 	 * as the driver can always deal with an error in any dma or
4037 	 * access handle, we can just return the fme_status value.
4038 	 */
4039 	pci_ereport_post(dip, err, NULL);
4040 	return (err->fme_status);
4041 }
4042 
4043 static void
4044 ixgbe_fm_init(ixgbe_t *ixgbe)
4045 {
4046 	ddi_iblock_cookie_t iblk;
4047 	int fma_acc_flag, fma_dma_flag;
4048 
4049 	/*
4050 	 * Only register with IO Fault Services if we have some capability
4051 	 */
4052 	if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
4053 		ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
4054 		fma_acc_flag = 1;
4055 	} else {
4056 		ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
4057 		fma_acc_flag = 0;
4058 	}
4059 
4060 	if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
4061 		fma_dma_flag = 1;
4062 	} else {
4063 		fma_dma_flag = 0;
4064 	}
4065 
4066 	ixgbe_set_fma_flags(fma_acc_flag, fma_dma_flag);
4067 
4068 	if (ixgbe->fm_capabilities) {
4069 
4070 		/*
4071 		 * Register capabilities with IO Fault Services
4072 		 */
4073 		ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk);
4074 
4075 		/*
4076 		 * Initialize pci ereport capabilities if ereport capable
4077 		 */
4078 		if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
4079 		    DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
4080 			pci_ereport_setup(ixgbe->dip);
4081 
4082 		/*
4083 		 * Register error callback if error callback capable
4084 		 */
4085 		if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
4086 			ddi_fm_handler_register(ixgbe->dip,
4087 			    ixgbe_fm_error_cb, (void*) ixgbe);
4088 	}
4089 }
4090 
4091 static void
4092 ixgbe_fm_fini(ixgbe_t *ixgbe)
4093 {
4094 	/*
4095 	 * Only unregister FMA capabilities if they are registered
4096 	 */
4097 	if (ixgbe->fm_capabilities) {
4098 
4099 		/*
4100 		 * Release any resources allocated by pci_ereport_setup()
4101 		 */
4102 		if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
4103 		    DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
4104 			pci_ereport_teardown(ixgbe->dip);
4105 
4106 		/*
4107 		 * Un-register error callback if error callback capable
4108 		 */
4109 		if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
4110 			ddi_fm_handler_unregister(ixgbe->dip);
4111 
4112 		/*
4113 		 * Unregister from IO Fault Service
4114 		 */
4115 		ddi_fm_fini(ixgbe->dip);
4116 	}
4117 }
4118 
4119 void
4120 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail)
4121 {
4122 	uint64_t ena;
4123 	char buf[FM_MAX_CLASS];
4124 
4125 	(void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
4126 	ena = fm_ena_generate(0, FM_ENA_FMT1);
4127 	if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) {
4128 		ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP,
4129 		    FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
4130 	}
4131 }
4132 
4133 static int
4134 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num)
4135 {
4136 	ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh;
4137 
4138 	mutex_enter(&rx_ring->rx_lock);
4139 	rx_ring->ring_gen_num = mr_gen_num;
4140 	mutex_exit(&rx_ring->rx_lock);
4141 	return (0);
4142 }
4143 
4144 /*
4145  * Callback funtion for MAC layer to register all rings.
4146  */
4147 /* ARGSUSED */
4148 void
4149 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int rg_index,
4150     const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh)
4151 {
4152 	ixgbe_t *ixgbe = (ixgbe_t *)arg;
4153 	mac_intr_t *mintr = &infop->mri_intr;
4154 
4155 	switch (rtype) {
4156 	case MAC_RING_TYPE_RX: {
4157 		ASSERT(rg_index == 0);
4158 		ASSERT(ring_index < ixgbe->num_rx_rings);
4159 
4160 		ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[ring_index];
4161 		rx_ring->ring_handle = rh;
4162 
4163 		infop->mri_driver = (mac_ring_driver_t)rx_ring;
4164 		infop->mri_start = ixgbe_ring_start;
4165 		infop->mri_stop = NULL;
4166 		infop->mri_poll = ixgbe_ring_rx_poll;
4167 
4168 		mintr->mi_handle = (mac_intr_handle_t)rx_ring;
4169 		mintr->mi_enable = ixgbe_rx_ring_intr_enable;
4170 		mintr->mi_disable = ixgbe_rx_ring_intr_disable;
4171 
4172 		break;
4173 	}
4174 	case MAC_RING_TYPE_TX: {
4175 		ASSERT(rg_index == -1);
4176 		ASSERT(ring_index < ixgbe->num_tx_rings);
4177 
4178 		ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index];
4179 		tx_ring->ring_handle = rh;
4180 
4181 		infop->mri_driver = (mac_ring_driver_t)tx_ring;
4182 		infop->mri_start = NULL;
4183 		infop->mri_stop = NULL;
4184 		infop->mri_tx = ixgbe_ring_tx;
4185 
4186 		break;
4187 	}
4188 	default:
4189 		break;
4190 	}
4191 }
4192 
4193 /*
4194  * Callback funtion for MAC layer to register all groups.
4195  */
4196 void
4197 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index,
4198     mac_group_info_t *infop, mac_group_handle_t gh)
4199 {
4200 	ixgbe_t *ixgbe = (ixgbe_t *)arg;
4201 
4202 	switch (rtype) {
4203 	case MAC_RING_TYPE_RX: {
4204 		ixgbe_rx_group_t *rx_group;
4205 
4206 		rx_group = &ixgbe->rx_groups[index];
4207 		rx_group->group_handle = gh;
4208 
4209 		infop->mgi_driver = (mac_group_driver_t)rx_group;
4210 		infop->mgi_start = NULL;
4211 		infop->mgi_stop = NULL;
4212 		infop->mgi_addmac = ixgbe_addmac;
4213 		infop->mgi_remmac = ixgbe_remmac;
4214 		infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups);
4215 
4216 		break;
4217 	}
4218 	case MAC_RING_TYPE_TX:
4219 		break;
4220 	default:
4221 		break;
4222 	}
4223 }
4224 
4225 /*
4226  * Enable interrupt on the specificed rx ring.
4227  */
4228 int
4229 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh)
4230 {
4231 	ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
4232 	ixgbe_t *ixgbe = rx_ring->ixgbe;
4233 	int r_idx = rx_ring->index;
4234 	int v_idx = rx_ring->intr_vector;
4235 
4236 	mutex_enter(&ixgbe->gen_lock);
4237 	ASSERT(BT_TEST(ixgbe->vect_map[v_idx].rx_map, r_idx) == 0);
4238 
4239 	/*
4240 	 * To enable interrupt by setting the VAL bit of given interrupt
4241 	 * vector allocation register (IVAR).
4242 	 */
4243 	ixgbe_enable_ivar(ixgbe, IXGBE_IVAR_RX_QUEUE(r_idx));
4244 
4245 	BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
4246 	mutex_exit(&ixgbe->gen_lock);
4247 
4248 	return (0);
4249 }
4250 
4251 /*
4252  * Disable interrupt on the specificed rx ring.
4253  */
4254 int
4255 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh)
4256 {
4257 	ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
4258 	ixgbe_t *ixgbe = rx_ring->ixgbe;
4259 	int r_idx = rx_ring->index;
4260 	int v_idx = rx_ring->intr_vector;
4261 
4262 	mutex_enter(&ixgbe->gen_lock);
4263 
4264 	ASSERT(BT_TEST(ixgbe->vect_map[v_idx].rx_map, r_idx) == 1);
4265 
4266 	/*
4267 	 * To disable interrupt by clearing the VAL bit of given interrupt
4268 	 * vector allocation register (IVAR).
4269 	 */
4270 	ixgbe_disable_ivar(ixgbe, IXGBE_IVAR_RX_QUEUE(r_idx));
4271 
4272 	BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx);
4273 
4274 	mutex_exit(&ixgbe->gen_lock);
4275 
4276 	return (0);
4277 }
4278 
4279 /*
4280  * Add a mac address.
4281  */
4282 static int
4283 ixgbe_addmac(void *arg, const uint8_t *mac_addr)
4284 {
4285 	ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
4286 	ixgbe_t *ixgbe = rx_group->ixgbe;
4287 	int slot;
4288 	int err;
4289 
4290 	mutex_enter(&ixgbe->gen_lock);
4291 
4292 	if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
4293 		mutex_exit(&ixgbe->gen_lock);
4294 		return (ECANCELED);
4295 	}
4296 
4297 	if (ixgbe->unicst_avail == 0) {
4298 		/* no slots available */
4299 		mutex_exit(&ixgbe->gen_lock);
4300 		return (ENOSPC);
4301 	}
4302 
4303 	for (slot = 0; slot < ixgbe->unicst_total; slot++) {
4304 		if (ixgbe->unicst_addr[slot].mac.set == 0)
4305 			break;
4306 	}
4307 
4308 	ASSERT((slot >= 0) && (slot < ixgbe->unicst_total));
4309 
4310 	if ((err = ixgbe_unicst_set(ixgbe, mac_addr, slot)) == 0) {
4311 		ixgbe->unicst_addr[slot].mac.set = 1;
4312 		ixgbe->unicst_avail--;
4313 	}
4314 
4315 	mutex_exit(&ixgbe->gen_lock);
4316 
4317 	return (err);
4318 }
4319 
4320 /*
4321  * Remove a mac address.
4322  */
4323 static int
4324 ixgbe_remmac(void *arg, const uint8_t *mac_addr)
4325 {
4326 	ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
4327 	ixgbe_t *ixgbe = rx_group->ixgbe;
4328 	int slot;
4329 	int err;
4330 
4331 	mutex_enter(&ixgbe->gen_lock);
4332 
4333 	if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
4334 		mutex_exit(&ixgbe->gen_lock);
4335 		return (ECANCELED);
4336 	}
4337 
4338 	slot = ixgbe_unicst_find(ixgbe, mac_addr);
4339 	if (slot == -1) {
4340 		mutex_exit(&ixgbe->gen_lock);
4341 		return (EINVAL);
4342 	}
4343 
4344 	if (ixgbe->unicst_addr[slot].mac.set == 0) {
4345 		mutex_exit(&ixgbe->gen_lock);
4346 		return (EINVAL);
4347 	}
4348 
4349 	bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
4350 	if ((err = ixgbe_unicst_set(ixgbe,
4351 	    ixgbe->unicst_addr[slot].mac.addr, slot)) == 0) {
4352 		ixgbe->unicst_addr[slot].mac.set = 0;
4353 		ixgbe->unicst_avail++;
4354 	}
4355 
4356 	mutex_exit(&ixgbe->gen_lock);
4357 
4358 	return (err);
4359 }
4360